public void RemovalNotificationReplacedEntry()
{
int key = ThreadLocalRandom.Current.Next();
const string value = nameof(RemovalNotificationReplacedEntry);
bool removalCalled = false;
Action <RemovalNotification <int, string> > removalNotifier = notification =>
{
/* We should only be called once */
Assert.IsFalse(removalCalled);
removalCalled = true;
Assert.AreEqual(key, notification.Key);
Assert.AreEqual(value, notification.Value);
Assert.AreEqual(RemovalCause.Replaced, notification.RemovalCause);
};
ICache <int, string> arbitraryCache =
CacheBuilder <int, string> .NewBuilder().WithRemovalListener(removalNotifier).Build();
arbitraryCache.Put(key, value);
const string differentValue = value + "_overridden";
arbitraryCache.Put(key, differentValue);
Assert.IsTrue(removalCalled);
}
public void SingleThreadRandomAccessLruExpiration()
{
const int maxElements = 10;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string value = nameof(SingleThreadRandomAccessLruExpiration);
for (int i = 0; i < maxElements * 10; ++i)
{
cache.Put(i, value);
/* This should make it so that the first elements in the cache are the warmest, always, and will never be removed */
for (int j = 0; (j < i) && (j < maxElements - 1); ++j)
{
string doesntMatter;
bool exists = cache.GetIfPresent(j, out doesntMatter);
Assert.IsTrue(exists);
}
for (int j = maxElements; j < i; ++j)
{
string shouldntExist;
bool itExistedOhNo = cache.GetIfPresent(j, out shouldntExist);
Assert.IsFalse(itExistedOhNo);
}
}
}
public void NoDataRaceOnKeyCollision()
{
Action testFunction = () =>
{
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().Build();
int seed = ThreadLocalRandom.Current.Next(10, 500);
int originalSeed = seed;
int key = ThreadLocalRandom.Current.Next();
List <Task> readers = new List <Task>(MaxThreads);
for (int i = 0; i < MaxThreads; ++i)
{
Task reader = new Task(() => cache.Get(key, _ => Interlocked.Add(ref seed, 1).ToString()));
readers.Add(reader);
reader.Start();
}
readers.ForEach(reader => reader.Wait());
Assert.AreEqual(cache.Count, 1);
string foundValue;
bool found = cache.GetIfPresent(key, out foundValue);
Assert.IsTrue(found);
Assert.AreEqual(originalSeed + 1, seed);
Assert.AreEqual(foundValue, (originalSeed + 1).ToString());
};
testFunction.RunMultipleTimes(NumRuntimes);
}
public void LruCacheHandlesDuplicateKeysCorrectly()
{
const int maxElements = 10;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string value = nameof(LruCacheHandlesDuplicateKeysCorrectly) + "_original";
const string overriddenValue = nameof(LruCacheHandlesDuplicateKeysCorrectly) + "_overridden";
for (int i = 0; i < maxElements * 10; ++i)
{
cache.Put(i, value);
cache.Put(i, overriddenValue);
/* Cache eviction */
for (int j = 0; (j < i - maxElements) && (0 <= j); ++j)
{
string shouldNotExist;
bool exists = cache.GetIfPresent(j, out shouldNotExist);
Assert.IsFalse(exists);
}
int minKey = i - maxElements + 1;
/* Overridden values */
for (int j = minKey; (0 <= j) && (j <= i); ++j)
{
string shouldExist;
bool exists = cache.GetIfPresent(j, out shouldExist);
Assert.IsTrue(exists);
Assert.AreEqual(shouldExist, overriddenValue);
}
}
}
public void RemovalNotificationExpiredWriteTimeout()
{
int key = ThreadLocalRandom.Current.Next();
const string value = nameof(RemovalNotificationExpiredWriteTimeout);
bool removalCalled = false;
Action <RemovalNotification <int, string> > removalNotifier = notification =>
{
/* We should only be called once */
Assert.IsFalse(removalCalled);
removalCalled = true;
Assert.AreEqual(key, notification.Key);
Assert.AreEqual(value, notification.Value);
Assert.AreEqual(RemovalCause.Expired, notification.RemovalCause);
};
TimeSpan writeExpiry = TimeSpan.FromMilliseconds(ThreadLocalRandom.Current.Next(1000, 1500));
ICache <int, string> arbitraryCache =
CacheBuilder <int, string> .NewBuilder()
.WithRemovalListener(removalNotifier)
.WithExpireAfterWrite(writeExpiry)
.Build();
arbitraryCache.Put(key, value);
Thread.Sleep((int)writeExpiry.TotalMilliseconds * 2);
string outValue;
bool foundValue = arbitraryCache.GetIfPresent(key, out outValue);
Assert.IsFalse(foundValue);
/* Force a removal notification via a "modification" operation */
arbitraryCache.Put(key, value + "_overridden");
Assert.IsTrue(removalCalled);
}
public void NoRemovalNotificationOnPutSameValue()
{
HashSet <int> expiredKeys = new HashSet <int>();
Action <RemovalNotification <int, string> > removalNotifier = notification =>
{
bool newKey = expiredKeys.Add(notification.Key);
Assert.IsTrue(newKey);
};
const int maxElements = 10;
ICache <int, string> cache =
CacheBuilder <int, string> .NewBuilder()
.WithMaxElements(maxElements)
.WithRemovalListener(removalNotifier)
.Build();
const string value = nameof(NoRemovalNotificationOnPutSameValue);
for (int i = 0; i < maxElements; ++i)
{
cache.Put(i, value);
cache.Put(i, value);
string foundValue;
bool exists = cache.GetIfPresent(i, out foundValue);
Assert.IsTrue(exists);
Assert.AreSame(foundValue, value);
Assert.IsFalse(expiredKeys.Any());
}
}
public void ExpireAfterAccessRespectsTimeout()
{
TimeSpan accessExpiry = TimeSpan.FromMilliseconds(ThreadLocalRandom.Current.Next(100, 150));
ICache <int, string> arbitraryCache =
CacheBuilder <int, string> .NewBuilder().WithExpireAfterAccess(accessExpiry).Build();
string outValue;
bool foundValue;
int key = ThreadLocalRandom.Current.Next();
const string value = nameof(ExpireAfterAccessRespectsTimeout);
arbitraryCache.Put(key, value);
int maxAttempts = ThreadLocalRandom.Current.Next(5, 200);
for (int i = 0; i < maxAttempts; ++i)
{
foundValue = arbitraryCache.GetIfPresent(key, out outValue);
Assert.IsTrue(foundValue);
Assert.AreEqual(value, outValue);
/* Sleep for a little bit */
Thread.Sleep((int)(accessExpiry.TotalMilliseconds / 4));
}
/* Sleep for a lot */
Thread.Sleep((int)(accessExpiry.TotalMilliseconds * 2));
foundValue = arbitraryCache.GetIfPresent(key, out outValue);
Assert.IsFalse(foundValue);
}
public void SingleThreadLruEvictedNotification()
{
HashSet <int> expiredKeys = new HashSet <int>();
Action <RemovalNotification <int, string> > removalNotifier = notification =>
{
Assert.AreEqual(RemovalCause.Evicted, notification.RemovalCause);
bool newKey = expiredKeys.Add(notification.Key);
Assert.IsTrue(newKey);
};
const int maxElements = 10;
ICache <int, string> cache =
CacheBuilder <int, string> .NewBuilder()
.WithMaxElements(maxElements)
.WithRemovalListener(removalNotifier)
.Build();
const string value = nameof(SingleThreadLruEvictedNotification);
for (int i = 0; i < maxElements * 10; ++i)
{
cache.Put(i, value);
string lruValue;
int lruKey = i - maxElements;
bool exists = cache.GetIfPresent(lruKey, out lruValue);
Assert.IsFalse(exists);
Assert.IsTrue(cache.Count <= maxElements);
if (0 <= lruKey)
{
Assert.IsTrue(expiredKeys.Contains(lruKey));
}
}
}
public Groups(Configuration conf, Timer timer)
{
impl = ReflectionUtils.NewInstance(conf.GetClass <GroupMappingServiceProvider>(CommonConfigurationKeys
.HadoopSecurityGroupMapping, typeof(ShellBasedUnixGroupsMapping)), conf);
cacheTimeout = conf.GetLong(CommonConfigurationKeys.HadoopSecurityGroupsCacheSecs
, CommonConfigurationKeys.HadoopSecurityGroupsCacheSecsDefault) * 1000;
negativeCacheTimeout = conf.GetLong(CommonConfigurationKeys.HadoopSecurityGroupsNegativeCacheSecs
, CommonConfigurationKeys.HadoopSecurityGroupsNegativeCacheSecsDefault) * 1000;
warningDeltaMs = conf.GetLong(CommonConfigurationKeys.HadoopSecurityGroupsCacheWarnAfterMs
, CommonConfigurationKeys.HadoopSecurityGroupsCacheWarnAfterMsDefault);
ParseStaticMapping(conf);
this.timer = timer;
this.cache = CacheBuilder.NewBuilder().RefreshAfterWrite(cacheTimeout, TimeUnit.Milliseconds
).Ticker(new Groups.TimerToTickerAdapter(timer)).ExpireAfterWrite(10 * cacheTimeout
, TimeUnit.Milliseconds).Build(new Groups.GroupCacheLoader(this));
if (negativeCacheTimeout > 0)
{
Com.Google.Common.Cache.Cache <string, bool> tempMap = CacheBuilder.NewBuilder().ExpireAfterWrite
(negativeCacheTimeout, TimeUnit.Milliseconds).Ticker(new Groups.TimerToTickerAdapter
(timer)).Build();
negativeCache = Collections.NewSetFromMap(tempMap.AsMap());
}
if (Log.IsDebugEnabled())
{
Log.Debug("Group mapping impl=" + impl.GetType().FullName + "; cacheTimeout=" + cacheTimeout
+ "; warningDeltaMs=" + warningDeltaMs);
}
}
public void LruCacheCanCacheElementsAfterFullInvalidation()
{
const int maxElements = 100;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string valueBase = nameof(LruCacheCanCacheElementsAfterFullInvalidation);
for (int i = 0; i < maxElements; ++i)
{
string value = valueBase + i;
cache.Put(i, value);
}
cache.InvalidateAll();
for (int i = 0; i < maxElements * 10; ++i)
{
string value = valueBase + i;
cache.Put(i, value);
string retrievedValue;
bool success = cache.GetIfPresent(i, out retrievedValue);
Assert.IsTrue(success);
Assert.AreSame(value, retrievedValue);
}
}
public void ConcurrentAccessDoesntLockUp()
{
Action testFunction = () =>
{
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().Build();
List <Task> readers = new List <Task>(MaxThreads);
for (int i = 0; i < MaxThreads; ++i)
{
int initialKey = i;
Task reader = new Task(() => cache.Get(initialKey, key => key.ToString()));
readers.Add(reader);
reader.Start();
}
readers.ForEach(reader => reader.Wait());
for (int i = 0; i < MaxThreads; ++i)
{
string foundValue;
bool found = cache.GetIfPresent(i, out foundValue);
Assert.IsTrue(found, "Didn't find " + i);
Assert.AreEqual(i.ToString(), foundValue);
}
};
testFunction.RunMultipleTimes(NumRuntimes);
}
public void LruCacheInsertsElements()
{
const int maxElements = 100;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string valueBase = nameof(LruCacheInsertsElements);
for (int i = 0; i < maxElements * 10; ++i)
{
string value = valueBase + i;
cache.Put(i, value);
string retrievedValue;
bool success = cache.GetIfPresent(i, out retrievedValue);
Assert.IsTrue(success);
Assert.AreSame(value, retrievedValue);
for (int j = i - maxElements + 1; (j <= i) && (0 <= j); ++j)
{
string otherCacheValue;
bool exists = cache.GetIfPresent(j, out otherCacheValue);
Assert.IsTrue(exists);
string expected = valueBase + j;
Assert.AreEqual(expected, otherCacheValue);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.ExecutionException"/>
public virtual void TestDownloadPublicWithStatCache()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
// if test directory doesn't have ancestor permission, skip this test
FileSystem f = basedir.GetFileSystem(conf);
Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
int size = 512;
ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger
>();
CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader
(conf);
LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build
(new _CacheLoader_328(counts, loader));
// increment the count
// use the default loader
// test FSDownload.isPublic() concurrently
int fileCount = 3;
IList <Callable <bool> > tasks = new AList <Callable <bool> >();
for (int i = 0; i < fileCount; i++)
{
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
Path path = new Path(basedir, "test-file-" + i);
CreateFile(files, path, size, rand);
FileSystem fs = path.GetFileSystem(conf);
FileStatus sStat = fs.GetFileStatus(path);
tasks.AddItem(new _Callable_358(fs, path, sStat, statCache));
}
ExecutorService exec = Executors.NewFixedThreadPool(fileCount);
try
{
IList <Future <bool> > futures = exec.InvokeAll(tasks);
// files should be public
foreach (Future <bool> future in futures)
{
NUnit.Framework.Assert.IsTrue(future.Get());
}
// for each path exactly one file status call should be made
foreach (AtomicInteger count in counts.Values)
{
NUnit.Framework.Assert.AreSame(count.Get(), 1);
}
}
finally
{
exec.Shutdown();
}
}
/// <summary>Create a new KMSAudit.</summary>
/// <param name="windowMs">
/// Duplicate events within the aggregation window are quashed
/// to reduce log traffic. A single message for aggregated
/// events is printed at the end of the window, along with a
/// count of the number of aggregated events.
/// </param>
internal KMSAudit(long windowMs)
{
cache = CacheBuilder.NewBuilder().ExpireAfterWrite(windowMs, TimeUnit.Milliseconds
).RemovalListener(new _RemovalListener_118(this)).Build();
executor = Executors.NewScheduledThreadPool(1, new ThreadFactoryBuilder().SetDaemon
(true).SetNameFormat(KmsLoggerName + "_thread").Build());
executor.ScheduleAtFixedRate(new _Runnable_132(this), windowMs / 10, windowMs / 10
, TimeUnit.Milliseconds);
}
public void RemovalListenerSet()
{
CacheBuilder <int, string> cacheBuilder = CacheBuilder <int, string> .NewBuilder();
Action <RemovalNotification <int, string> > notification = removalNotification => { };
cacheBuilder.WithRemovalListener(notification);
Assert.IsNotNull(cacheBuilder.RemovalListener);
Assert.AreEqual(notification, cacheBuilder.RemovalListener);
}
public void TestExpireAfterWriteSet()
{
CacheBuilder <int, string> cacheBuilder = CacheBuilder <int, string> .NewBuilder();
TimeSpan expireAfterWriteSeconds = TimeSpan.FromSeconds(ThreadLocalRandom.Current.Next());
cacheBuilder.WithExpireAfterAccess(expireAfterWriteSeconds);
Assert.AreEqual((long)Math.Round(expireAfterWriteSeconds.TotalMilliseconds),
cacheBuilder.ExpireAfterAccessMilliseconds);
}
internal CacheExtension(KeyProvider prov, long keyTimeoutMillis, long currKeyTimeoutMillis
)
{
this.provider = prov;
keyVersionCache = CacheBuilder.NewBuilder().ExpireAfterAccess(keyTimeoutMillis, TimeUnit
.Milliseconds).Build(new _CacheLoader_49(this));
keyMetadataCache = CacheBuilder.NewBuilder().ExpireAfterAccess(keyTimeoutMillis,
TimeUnit.Milliseconds).Build(new _CacheLoader_62(this));
currentKeyCache = CacheBuilder.NewBuilder().ExpireAfterWrite(currKeyTimeoutMillis
, TimeUnit.Milliseconds).Build(new _CacheLoader_75(this));
}
internal DFSClientCache(NfsConfiguration config, int clientCache)
{
this.config = config;
this.clientCache = CacheBuilder.NewBuilder().MaximumSize(clientCache).RemovalListener
(ClientRemovalListener()).Build(ClientLoader());
this.inputstreamCache = CacheBuilder.NewBuilder().MaximumSize(DefaultDfsInputstreamCacheSize
).ExpireAfterAccess(DefaultDfsInputstreamCacheTtl, TimeUnit.Seconds).RemovalListener
(InputStreamRemovalListener()).Build(InputStreamLoader());
ShutdownHookManager.Get().AddShutdownHook(new DFSClientCache.CacheFinalizer(this)
, ShutdownHookPriority);
}
public void SingleThreadLruExpiration()
{
const int maxElements = 10;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string value = nameof(SingleThreadLruExpiration);
for (int i = 0; i < maxElements * 10; ++i)
{
cache.Put(i, value);
string lruValue;
bool exists = cache.GetIfPresent(i - maxElements, out lruValue);
Assert.IsFalse(exists);
Assert.IsTrue(cache.Count <= maxElements);
}
}
public void NoRemovalNotificationSameEntry()
{
bool removalCalled = false;
Action <RemovalNotification <int, string> > removalNotifier =
notification =>
{
Assert.Fail("Removal notification was called when calling put with same key value pair");
};
ICache <int, string> arbitraryCache =
CacheBuilder <int, string> .NewBuilder().WithRemovalListener(removalNotifier).Build();
int key = ThreadLocalRandom.Current.Next();
const string value = nameof(NoRemovalNotificationSameEntry);
arbitraryCache.Put(key, value);
arbitraryCache.Put(key, value);
Assert.IsFalse(removalCalled);
}
public void ExpireAfterWriteRespectsTimeout()
{
TimeSpan writeExpiry = TimeSpan.FromMilliseconds(ThreadLocalRandom.Current.Next(100, 150));
ICache <int, string> arbitraryCache =
CacheBuilder <int, string> .NewBuilder().WithExpireAfterWrite(writeExpiry).Build();
int key = ThreadLocalRandom.Current.Next();
const string value = nameof(ExpireAfterWriteRespectsTimeout);
arbitraryCache.Put(key, value);
string outValue;
bool foundValue = arbitraryCache.GetIfPresent(key, out outValue);
Assert.IsTrue(foundValue);
Assert.AreEqual(value, outValue);
Thread.Sleep((int)(writeExpiry.TotalMilliseconds * 2));
foundValue = arbitraryCache.GetIfPresent(key, out outValue);
Assert.IsFalse(foundValue);
}
public void LruCacheInvalidatesAllElements()
{
const int maxElements = 100;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string valueBase = nameof(LruCacheInvalidatesAllElements);
for (int i = 0; i < maxElements; ++i)
{
string value = valueBase + i;
cache.Put(i, value);
}
cache.InvalidateAll();
for (int i = 0; i < maxElements; ++i)
{
string value;
bool valueExists = cache.GetIfPresent(i, out value);
Assert.IsFalse(valueExists);
}
}
/// <summary>Constructor takes the following tunable configuration parameters</summary>
/// <param name="numValues">
/// The number of values cached in the Queue for a
/// particular key.
/// </param>
/// <param name="lowWatermark">
/// The ratio of (number of current entries/numValues)
/// below which the <code>fillQueueForKey()</code> funciton will be
/// invoked to fill the Queue.
/// </param>
/// <param name="expiry">
/// Expiry time after which the Key and associated Queue are
/// evicted from the cache.
/// </param>
/// <param name="numFillerThreads">Number of threads to use for the filler thread</param>
/// <param name="policy">
/// The SyncGenerationPolicy to use when client
/// calls "getAtMost"
/// </param>
/// <param name="refiller">implementation of the QueueRefiller</param>
public ValueQueue(int numValues, float lowWatermark, long expiry, int numFillerThreads
, ValueQueue.SyncGenerationPolicy policy, ValueQueue.QueueRefiller <E> refiller)
{
// Return atleast 1 value
// Return min(n, lowWatermark * numValues) values
// Return n values
Preconditions.CheckArgument(numValues > 0, "\"numValues\" must be > 0");
Preconditions.CheckArgument(((lowWatermark > 0) && (lowWatermark <= 1)), "\"lowWatermark\" must be > 0 and <= 1"
);
Preconditions.CheckArgument(expiry > 0, "\"expiry\" must be > 0");
Preconditions.CheckArgument(numFillerThreads > 0, "\"numFillerThreads\" must be > 0"
);
Preconditions.CheckNotNull(policy, "\"policy\" must not be null");
this.refiller = refiller;
this.policy = policy;
this.numValues = numValues;
this.lowWatermark = lowWatermark;
keyQueues = CacheBuilder.NewBuilder().ExpireAfterAccess(expiry, TimeUnit.Milliseconds
).Build(new _CacheLoader_175(refiller, lowWatermark, numValues));
executor = new ThreadPoolExecutor(numFillerThreads, numFillerThreads, 0L, TimeUnit
.Milliseconds, queue, new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat(RefillThread
).Build());
}
public void LruCacheInvalidatesElements()
{
const int maxElements = 100;
ICache <int, string> cache = CacheBuilder <int, string> .NewBuilder().WithMaxElements(maxElements).Build();
const string valueBase = nameof(LruCacheInvalidatesElements);
for (int i = 0; i < maxElements * 10; ++i)
{
string value = valueBase + i;
cache.Put(i, value);
string retrievedValue;
bool success = cache.GetIfPresent(i, out retrievedValue);
Assert.IsTrue(success);
Assert.AreSame(value, retrievedValue);
cache.Invalidate(i);
bool shouldFail = cache.GetIfPresent(i, out retrievedValue);
Assert.IsFalse(shouldFail);
}
Assert.AreEqual(0, cache.Count);
}
private static LoadingCache <Type, AtomicInteger> CreateCache <T>()
{
System.Type klass = typeof(T);
return(CacheBuilder.NewBuilder().Build(new _CacheLoader_63()));
}
public void RemovalListenerUnsetIsNull()
{
CacheBuilder <int, string> cacheBuilder = CacheBuilder <int, string> .NewBuilder();
Assert.IsNull(cacheBuilder.RemovalListener);
}
public KeyProviderCache(long expiryMs)
{
cache = CacheBuilder.NewBuilder().ExpireAfterAccess(expiryMs, TimeUnit.Milliseconds
).RemovalListener(new _RemovalListener_47()).Build();
}
public void InvalidExpireNullRemovalListenerThrows()
{
CacheBuilder <int, string> .NewBuilder().WithRemovalListener(null);
}
public void InvalidExpireAfterWriteThrows()
{
CacheBuilder <int, string> .NewBuilder().WithExpireAfterWrite(TimeSpan.FromSeconds(-1));
}
public void ExpireAfterWriteUnsetTurnsToDefault()
{
CacheBuilder <int, string> cacheBuilder = CacheBuilder <int, string> .NewBuilder();
Assert.IsNull(cacheBuilder.ExpireAfterWriteMilliseconds);
}