/// <summary>
/// Obtains the file status, first by checking the stat cache if it is
/// available, and then by getting it explicitly from the filesystem.
/// </summary>
/// <remarks>
/// Obtains the file status, first by checking the stat cache if it is
/// available, and then by getting it explicitly from the filesystem. If we got
/// the file status from the filesystem, it is added to the stat cache.
/// The stat cache is expected to be managed by callers who provided it to
/// FSDownload.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private static FileStatus GetFileStatus(FileSystem fs, Path path, LoadingCache <Path
, Future <FileStatus> > statCache)
{
// if the stat cache does not exist, simply query the filesystem
if (statCache == null)
{
return(fs.GetFileStatus(path));
}
try
{
// get or load it from the cache
return(statCache.Get(path).Get());
}
catch (ExecutionException e)
{
Exception cause = e.InnerException;
// the underlying exception should normally be IOException
if (cause is IOException)
{
throw (IOException)cause;
}
else
{
throw new IOException(cause);
}
}
catch (Exception e)
{
// should not happen
Sharpen.Thread.CurrentThread().Interrupt();
throw new IOException(e);
}
}
public Groups(Configuration conf, Timer timer)
{
impl = ReflectionUtils.NewInstance(conf.GetClass <GroupMappingServiceProvider>(CommonConfigurationKeys
.HadoopSecurityGroupMapping, typeof(ShellBasedUnixGroupsMapping)), conf);
cacheTimeout = conf.GetLong(CommonConfigurationKeys.HadoopSecurityGroupsCacheSecs
, CommonConfigurationKeys.HadoopSecurityGroupsCacheSecsDefault) * 1000;
negativeCacheTimeout = conf.GetLong(CommonConfigurationKeys.HadoopSecurityGroupsNegativeCacheSecs
, CommonConfigurationKeys.HadoopSecurityGroupsNegativeCacheSecsDefault) * 1000;
warningDeltaMs = conf.GetLong(CommonConfigurationKeys.HadoopSecurityGroupsCacheWarnAfterMs
, CommonConfigurationKeys.HadoopSecurityGroupsCacheWarnAfterMsDefault);
ParseStaticMapping(conf);
this.timer = timer;
this.cache = CacheBuilder.NewBuilder().RefreshAfterWrite(cacheTimeout, TimeUnit.Milliseconds
).Ticker(new Groups.TimerToTickerAdapter(timer)).ExpireAfterWrite(10 * cacheTimeout
, TimeUnit.Milliseconds).Build(new Groups.GroupCacheLoader(this));
if (negativeCacheTimeout > 0)
{
Com.Google.Common.Cache.Cache <string, bool> tempMap = CacheBuilder.NewBuilder().ExpireAfterWrite
(negativeCacheTimeout, TimeUnit.Milliseconds).Ticker(new Groups.TimerToTickerAdapter
(timer)).Build();
negativeCache = Collections.NewSetFromMap(tempMap.AsMap());
}
if (Log.IsDebugEnabled())
{
Log.Debug("Group mapping impl=" + impl.GetType().FullName + "; cacheTimeout=" + cacheTimeout
+ "; warningDeltaMs=" + warningDeltaMs);
}
}
public static IObservable <T> Load <T>(string assetPath) where T : UObject
{
if (_loadedAssetDict.ContainsKey(assetPath))
{
return(Observable.Return <T>(_loadedAssetDict[assetPath].asset as T));
}
if (_assetCacheDict.ContainsKey(assetPath))
{
return(Observable.Return <T>(_assetCacheDict[assetPath].Asset as T));
}
LoadingCache <UObject> loadingCache = null;
if (!_loadingCacheDict.TryGetValue(assetPath, out loadingCache))
{
loadingCache = new LoadingCache <UObject>();
_loadingCacheDict.Add(assetPath, loadingCache);
_assetLoader.Load <T>(assetPath)
.Do(_ =>
{
_loadedAssetDict.Add(assetPath, new LoadedAssetInfo()
{
asset = _, instance = null
});
_assetNameDict.Add(_, assetPath);
_loadingCacheDict.Remove(assetPath);
}).Subscribe(loadingCache);
}
return(loadingCache.ToLoadingObserable().Select(obj => obj as T));
}
public _Callable_358(FileSystem fs, Path path, FileStatus sStat, LoadingCache <Path
, Future <FileStatus> > statCache)
{
this.fs = fs;
this.path = path;
this.sStat = sStat;
this.statCache = statCache;
}
public LocalizerContext(string user, ContainerId containerId, Credentials credentials
, LoadingCache <Path, Future <FileStatus> > statCache)
{
this.user = user;
this.containerId = containerId;
this.credentials = credentials;
this.statCache = statCache;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.ExecutionException"/>
public virtual void TestDownloadPublicWithStatCache()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
// if test directory doesn't have ancestor permission, skip this test
FileSystem f = basedir.GetFileSystem(conf);
Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
int size = 512;
ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger
>();
CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader
(conf);
LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build
(new _CacheLoader_328(counts, loader));
// increment the count
// use the default loader
// test FSDownload.isPublic() concurrently
int fileCount = 3;
IList <Callable <bool> > tasks = new AList <Callable <bool> >();
for (int i = 0; i < fileCount; i++)
{
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
Path path = new Path(basedir, "test-file-" + i);
CreateFile(files, path, size, rand);
FileSystem fs = path.GetFileSystem(conf);
FileStatus sStat = fs.GetFileStatus(path);
tasks.AddItem(new _Callable_358(fs, path, sStat, statCache));
}
ExecutorService exec = Executors.NewFixedThreadPool(fileCount);
try
{
IList <Future <bool> > futures = exec.InvokeAll(tasks);
// files should be public
foreach (Future <bool> future in futures)
{
NUnit.Framework.Assert.IsTrue(future.Get());
}
// for each path exactly one file status call should be made
foreach (AtomicInteger count in counts.Values)
{
NUnit.Framework.Assert.AreSame(count.Get(), 1);
}
}
finally
{
exec.Shutdown();
}
}
/// <summary>
/// Checks for a given path whether the Other permissions on it
/// imply the permission in the passed FsAction
/// </summary>
/// <param name="fs"/>
/// <param name="path"/>
/// <param name="action"/>
/// <returns>true if the path in the uri is visible to all, false otherwise</returns>
/// <exception cref="System.IO.IOException"/>
private static bool CheckPermissionOfOther(FileSystem fs, Path path, FsAction action
, LoadingCache <Path, Future <FileStatus> > statCache)
{
FileStatus status = GetFileStatus(fs, path, statCache);
FsPermission perms = status.GetPermission();
FsAction otherAction = perms.GetOtherAction();
return(otherAction.Implies(action));
}
private static void UpdateLeaseCount <T>(LoadingCache <Type, AtomicInteger> usageCounts
, T codec, int delta)
{
if (codec != null)
{
Type codecClass = ReflectionUtils.GetClass(codec);
usageCounts.GetUnchecked(codecClass).AddAndGet(delta);
}
}
public FSDownload(FileContext files, UserGroupInformation ugi, Configuration conf
, Path destDirPath, LocalResource resource, LoadingCache <Path, Future <FileStatus
> > statCache)
{
this.conf = conf;
this.destDirPath = destDirPath;
this.files = files;
this.userUgi = ugi;
this.resource = resource;
this.statCache = statCache;
}
internal DFSClientCache(NfsConfiguration config, int clientCache)
{
this.config = config;
this.clientCache = CacheBuilder.NewBuilder().MaximumSize(clientCache).RemovalListener
(ClientRemovalListener()).Build(ClientLoader());
this.inputstreamCache = CacheBuilder.NewBuilder().MaximumSize(DefaultDfsInputstreamCacheSize
).ExpireAfterAccess(DefaultDfsInputstreamCacheTtl, TimeUnit.Seconds).RemovalListener
(InputStreamRemovalListener()).Build(InputStreamLoader());
ShutdownHookManager.Get().AddShutdownHook(new DFSClientCache.CacheFinalizer(this)
, ShutdownHookPriority);
}
internal CacheExtension(KeyProvider prov, long keyTimeoutMillis, long currKeyTimeoutMillis
)
{
this.provider = prov;
keyVersionCache = CacheBuilder.NewBuilder().ExpireAfterAccess(keyTimeoutMillis, TimeUnit
.Milliseconds).Build(new _CacheLoader_49(this));
keyMetadataCache = CacheBuilder.NewBuilder().ExpireAfterAccess(keyTimeoutMillis,
TimeUnit.Milliseconds).Build(new _CacheLoader_62(this));
currentKeyCache = CacheBuilder.NewBuilder().ExpireAfterWrite(currKeyTimeoutMillis
, TimeUnit.Milliseconds).Build(new _CacheLoader_75(this));
}
public static bool IsPublic(FileSystem fs, Path current, FileStatus sStat, LoadingCache
<Path, Future <FileStatus> > statCache)
{
current = fs.MakeQualified(current);
//the leaf level file should be readable by others
if (!CheckPublicPermsForAll(fs, sStat, FsAction.ReadExecute, FsAction.Read))
{
return(false);
}
if (Shell.Windows && fs is LocalFileSystem)
{
// Relax the requirement for public cache on LFS on Windows since default
// permissions are "700" all the way up to the drive letter. In this
// model, the only requirement for a user is to give EVERYONE group
// permission on the file and the file will be considered public.
// This code path is only hit when fs.default.name is file:/// (mainly
// in tests).
return(true);
}
return(AncestorsHaveExecutePermissions(fs, current.GetParent(), statCache));
}
/// <summary>Constructor takes the following tunable configuration parameters</summary>
/// <param name="numValues">
/// The number of values cached in the Queue for a
/// particular key.
/// </param>
/// <param name="lowWatermark">
/// The ratio of (number of current entries/numValues)
/// below which the <code>fillQueueForKey()</code> funciton will be
/// invoked to fill the Queue.
/// </param>
/// <param name="expiry">
/// Expiry time after which the Key and associated Queue are
/// evicted from the cache.
/// </param>
/// <param name="numFillerThreads">Number of threads to use for the filler thread</param>
/// <param name="policy">
/// The SyncGenerationPolicy to use when client
/// calls "getAtMost"
/// </param>
/// <param name="refiller">implementation of the QueueRefiller</param>
public ValueQueue(int numValues, float lowWatermark, long expiry, int numFillerThreads
, ValueQueue.SyncGenerationPolicy policy, ValueQueue.QueueRefiller <E> refiller)
{
// Return atleast 1 value
// Return min(n, lowWatermark * numValues) values
// Return n values
Preconditions.CheckArgument(numValues > 0, "\"numValues\" must be > 0");
Preconditions.CheckArgument(((lowWatermark > 0) && (lowWatermark <= 1)), "\"lowWatermark\" must be > 0 and <= 1"
);
Preconditions.CheckArgument(expiry > 0, "\"expiry\" must be > 0");
Preconditions.CheckArgument(numFillerThreads > 0, "\"numFillerThreads\" must be > 0"
);
Preconditions.CheckNotNull(policy, "\"policy\" must not be null");
this.refiller = refiller;
this.policy = policy;
this.numValues = numValues;
this.lowWatermark = lowWatermark;
keyQueues = CacheBuilder.NewBuilder().ExpireAfterAccess(expiry, TimeUnit.Milliseconds
).Build(new _CacheLoader_175(refiller, lowWatermark, numValues));
executor = new ThreadPoolExecutor(numFillerThreads, numFillerThreads, 0L, TimeUnit
.Milliseconds, queue, new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat(RefillThread
).Build());
}
internal static bool AncestorsHaveExecutePermissions(FileSystem fs, Path path, LoadingCache
<Path, Future <FileStatus> > statCache)
{
Path current = path;
while (current != null)
{
//the subdirs in the path should have execute permissions for others
if (!CheckPermissionOfOther(fs, current, FsAction.Execute, statCache))
{
return(false);
}
current = current.GetParent();
}
return(true);
}
private static int GetLeaseCount <T>(LoadingCache <Type, AtomicInteger> usageCounts
, Type codecClass)
{
return(usageCounts.GetUnchecked((Type)codecClass).Get());
}
