/// <returns>
/// a list of
/// <see cref="Org.Apache.Hadoop.FS.StorageType"/>
/// s for storing the replicas of a block.
/// </returns>
public virtual IList <StorageType> ChooseStorageTypes(short replication)
{
IList <StorageType> types = new List <StorageType>();
int i = 0;
int j = 0;
// Do not return transient storage types. We will not have accurate
// usage information for transient types.
for (; i < replication && j < storageTypes.Length; ++j)
{
if (!storageTypes[j].IsTransient())
{
types.AddItem(storageTypes[j]);
++i;
}
}
StorageType last = storageTypes[storageTypes.Length - 1];
if (!last.IsTransient())
{
for (; i < replication; i++)
{
types.AddItem(last);
}
}
return(types);
}
private bool CreateNoChecksumContext()
{
if (verifyChecksum)
{
if (storageType != null && storageType.IsTransient())
{
// Checksums are not stored for replicas on transient storage. We do not
// anchor, because we do not intend for client activity to block eviction
// from transient storage on the DataNode side.
return(true);
}
else
{
return(replica.AddNoChecksumAnchor());
}
}
else
{
return(true);
}
}
protected internal virtual ThreadPoolExecutor InitializeCacheExecutor(FilePath parent
)
{
if (storageType.IsTransient())
{
return(null);
}
if (dataset.datanode == null)
{
// FsVolumeImpl is used in test.
return(null);
}
int maxNumThreads = dataset.datanode.GetConf().GetInt(DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeKey
, DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeDefault);
ThreadFactory workerFactory = new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat
("FsVolumeImplWorker-" + parent.ToString() + "-%d").Build();
ThreadPoolExecutor executor = new ThreadPoolExecutor(1, maxNumThreads, 60, TimeUnit
.Seconds, new LinkedBlockingQueue <Runnable>(), workerFactory);
executor.AllowCoreThreadTimeOut(true);
return(executor);
}
/// <exception cref="System.IO.IOException"/>
private static BlockLocalPathInfo GetBlockPathInfo(UserGroupInformation ugi, ExtendedBlock
blk, DatanodeInfo node, Configuration conf, int timeout, Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier> token, bool connectToDnViaHostname, StorageType storageType
)
{
BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo
(node.GetIpcPort());
BlockLocalPathInfo pathinfo = null;
ClientDatanodeProtocol proxy = localDatanodeInfo.GetDatanodeProxy(ugi, node, conf
, timeout, connectToDnViaHostname);
try
{
// make RPC to local datanode to find local pathnames of blocks
pathinfo = proxy.GetBlockLocalPathInfo(blk, token);
// We cannot cache the path information for a replica on transient storage.
// If the replica gets evicted, then it moves to a different path. Then,
// our next attempt to read from the cached path would fail to find the
// file. Additionally, the failure would cause us to disable legacy
// short-circuit read for all subsequent use in the ClientContext. Unlike
// the newer short-circuit read implementation, we have no communication
// channel for the DataNode to notify the client that the path has been
// invalidated. Therefore, our only option is to skip caching.
if (pathinfo != null && !storageType.IsTransient())
{
if (Log.IsDebugEnabled())
{
Log.Debug("Cached location of block " + blk + " as " + pathinfo);
}
localDatanodeInfo.SetBlockLocalPathInfo(blk, pathinfo);
}
}
catch (IOException e)
{
localDatanodeInfo.ResetDatanodeProxy();
// Reset proxy on error
throw;
}
return(pathinfo);
}
// Multiple datanodes could be running on the local machine. Store proxies in
// a map keyed by the ipc port of the datanode.
// reader for the data file
// reader for the checksum file
/// <summary>The only way this object can be instantiated.</summary>
/// <exception cref="System.IO.IOException"/>
internal static BlockReaderLocalLegacy NewBlockReader(DFSClient.Conf conf, UserGroupInformation
userGroupInformation, Configuration configuration, string file, ExtendedBlock blk
, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, DatanodeInfo
node, long startOffset, long length, StorageType storageType)
{
BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo
(node.GetIpcPort());
// check the cache first
BlockLocalPathInfo pathinfo = localDatanodeInfo.GetBlockLocalPathInfo(blk);
if (pathinfo == null)
{
if (userGroupInformation == null)
{
userGroupInformation = UserGroupInformation.GetCurrentUser();
}
pathinfo = GetBlockPathInfo(userGroupInformation, blk, node, configuration, conf.
socketTimeout, token, conf.connectToDnViaHostname, storageType);
}
// check to see if the file exists. It may so happen that the
// HDFS file has been deleted and this block-lookup is occurring
// on behalf of a new HDFS file. This time, the block file could
// be residing in a different portion of the fs.data.dir directory.
// In this case, we remove this entry from the cache. The next
// call to this method will re-populate the cache.
FileInputStream dataIn = null;
FileInputStream checksumIn = null;
BlockReaderLocalLegacy localBlockReader = null;
bool skipChecksumCheck = conf.skipShortCircuitChecksums || storageType.IsTransient
();
try
{
// get a local file system
FilePath blkfile = new FilePath(pathinfo.GetBlockPath());
dataIn = new FileInputStream(blkfile);
if (Log.IsDebugEnabled())
{
Log.Debug("New BlockReaderLocalLegacy for file " + blkfile + " of size " + blkfile
.Length() + " startOffset " + startOffset + " length " + length + " short circuit checksum "
+ !skipChecksumCheck);
}
if (!skipChecksumCheck)
{
// get the metadata file
FilePath metafile = new FilePath(pathinfo.GetMetaPath());
checksumIn = new FileInputStream(metafile);
DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(new DataInputStream(
checksumIn), blk);
long firstChunkOffset = startOffset - (startOffset % checksum.GetBytesPerChecksum
());
localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset
, length, pathinfo, checksum, true, dataIn, firstChunkOffset, checksumIn);
}
else
{
localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset
, length, pathinfo, dataIn);
}
}
catch (IOException e)
{
// remove from cache
localDatanodeInfo.RemoveBlockLocalPathInfo(blk);
DFSClient.Log.Warn("BlockReaderLocalLegacy: Removing " + blk + " from cache because local file "
+ pathinfo.GetBlockPath() + " could not be opened.");
throw;
}
finally
{
if (localBlockReader == null)
{
if (dataIn != null)
{
dataIn.Close();
}
if (checksumIn != null)
{
checksumIn.Close();
}
}
}
return(localBlockReader);
}