public bool IsExpired(CacheRuntimeContext context) { if (_hint != null) { return(_hint.DetermineExpiration(context)); } return(false); }
/// <summary> /// Called by the scheduler to remove the items that has expired /// </summary> public bool Expire() { //indicates whether some items expired during this interval or not... bool expired = false; //if user has updated the file then the new values will be reloaded. _sleepInterval = ServiceConfiguration.ExpirationBulkRemoveDelay; _removeThreshhold = ServiceConfiguration.ExpirationBulkRemoveSize; CacheBase cacheInst = _context.CacheImpl; CacheBase cache = _context.CacheInternal; Cache rootCache = _context.CacheRoot; if (cache == null) { throw new InvalidOperationException("No cache instance defined"); } bool allowExpire = AllowClusteredExpiry; //in case of replication and por, only the coordinator/sub-coordinator is responsible to expire the items. if (!allowExpire) { return(false); } ClusteredArrayList selectedKeys = new ClusteredArrayList(); int oldItemsCount = 0; HashVector oldeItems = null; try { StartLogging(); DateTime startTime = DateTime.Now; int currentTime = AppUtil.DiffSeconds(startTime); int cleanSize = (int)Math.Ceiling(cache.Count * _cleanRatio); //set the flag that we are going to expire the items. if (_cacheLastAccessLoggingIntervalPassed >= _cacheLastAccessLoggingInterval) { _cacheLastAccessLoggingInterval = CacheLastAccessLoggingInterval; _cacheLastAccessCountEnabled = IsCacheLastAccessCountEnabled; _cacheLastAccessCountLoggingEnabled = IsCacheLastAccessLoggingEnabled; _cacheLastAccessInterval = CacheLastAccessCountInterval; } else { _cacheLastAccessLoggingIntervalPassed++; } if (_cacheLastAccessCountEnabled && _cacheLastAccessCountLoggingEnabled) { if (_cacheLastAccessLoggingIntervalPassed >= _cacheLastAccessLoggingInterval) { _cacheLastAccessLoggingIntervalPassed = 0; oldeItems = new HashVector(); } } lock (_mainIndex.SyncRoot) { IDictionaryEnumerator em = _mainIndex.GetEnumerator(); if (em != null) { while (em.MoveNext()) { ExpirationHint hint = em.Value as ExpirationHint; if (hint != null && _cacheLastAccessCountEnabled && hint is IdleExpiration) { IdleExpiration slidingExpHint = hint as IdleExpiration; TimeSpan diff = AppUtil.GetDateTime(AppUtil.DiffSeconds(DateTime.Now)) - AppUtil.GetDateTime(slidingExpHint.LastAccessTime); if (diff.TotalMinutes >= _cacheLastAccessInterval) { oldItemsCount++; if (oldeItems != null) { oldeItems.Add(em.Key, null); } } } if (hint == null || hint.SortKey.CompareTo(currentTime) >= 0) { continue; } if (hint.DetermineExpiration(_context)) { selectedKeys.Add(em.Key); if (cleanSize > 0 && selectedKeys.Count == cleanSize) { break; } } } } } if (NCacheLog.IsInfoEnabled) { NCacheLog.Info("ExpirationManager.Expire()", String.Format("Expiry time for {0}/{1} Items: " + (DateTime.UtcNow - startTime), selectedKeys.Count, /*_expiryIndex.KeyCount*/ cache.Count)); } } catch (Exception e) { NCacheLog.Error("ExpirationManager.Expire(bool)", "LocalCache(Expire): " + e.ToString()); } finally { _context.PerfStatsColl.IncrementCacheLastAccessCountStats(oldItemsCount); ApplyLoggs(); ClusteredArrayList dependentItems = new ClusteredArrayList(); DateTime startTime = DateTime.Now; HashVector expiredItemTable = new HashVector(); expiredItemTable.Add(ItemRemoveReason.Expired, selectedKeys);//Time based expiration try { IDictionaryEnumerator ide = expiredItemTable.GetEnumerator(); while (ide.MoveNext()) { selectedKeys = ide.Value as ClusteredArrayList; ItemRemoveReason removedReason = (ItemRemoveReason)ide.Key; if (selectedKeys.Count > 0) { //new architectural changes begins from here. ClusteredArrayList keysTobeRemoved = new ClusteredArrayList(); for (int i = 0; i < selectedKeys.Count && !_cacheCleared; i++) { keysTobeRemoved.Add(selectedKeys[i]); if (keysTobeRemoved.Count % _removeThreshhold == 0) { try { if (this.IsDisposed) { break; } OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); object[][] keysExposed = keysTobeRemoved.ToInternalArray(); foreach (object[] collection in keysExposed) { cache.RemoveSync(collection, removedReason, false, operationContext); } //set the flag that item has expired from cache... expired = true; if (_context.PerfStatsColl != null) { _context.PerfStatsColl.IncrementExpiryPerSecStatsBy(keysTobeRemoved.Count); } } catch (Exception e) { NCacheLog.Error("ExpiryManager.Expire", "an error occurred while removing expired items. Error " + e.ToString()); } keysTobeRemoved.Clear(); //we stop the activity of the current thread so that normal user operation is not affected. Thread.Sleep(_sleepInterval); } } if (!this.IsDisposed && keysTobeRemoved.Count > 0) { try { OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); object[][] keysExposed = keysTobeRemoved.ToInternalArray(); foreach (object[] keyCollection in keysExposed) { cache.RemoveSync(keyCollection, removedReason, false, operationContext); } //set the flag that item has expired from cache... expired = true; if (_context.PerfStatsColl != null) { _context.PerfStatsColl.IncrementExpiryPerSecStatsBy(keysTobeRemoved.Count); } } catch (Exception e) { NCacheLog.Error("ExpiryManager.Expire", "an error occurred while removing expired items. Error " + e.ToString()); } } } } } finally { _transitoryIndex.Clear(); lock (this) { _cacheCleared = false; } if (oldeItems != null) { StringBuilder sb = new StringBuilder(); IDictionaryEnumerator ide = oldeItems.GetEnumerator(); int count = 1; while (ide.MoveNext()) { sb.Append(ide.Key + ", "); if (count % 10 == 0) { sb.Append("\r\n"); count = 1; } else { count++; } } NCacheLog.Info(sb.ToString().Trim()); } } } return(expired); }
/// <summary> /// Called by the scheduler to remove the items that has expired /// </summary> public bool Expire() { //indicates whether some items expired during this interval or not... bool expired = false; //if user has updated the file then the new values will be reloaded. _sleepInterval = ServiceConfiguration.ExpirationBulkRemoveDelay; _removeThreshhold = ServiceConfiguration.ExpirationBulkRemoveSize; //notification is sent for a max of 100k data if multiple items... //otherwise if a single item is greater than 100k then notification is sent for //that item only... int notifThreshold = 30 * 1024; CacheBase cacheInst = _context.CacheImpl; CacheBase cache = _context.CacheInternal; Cache rootCache = _context.CacheRoot; object[] keys = null; object[] values = null; if (cache == null) { throw new InvalidOperationException("No cache instance defined"); } if (_context.ReaderMgr != null) { _context.ReaderMgr.ExpireReader(CleanInterval); } bool allowExpire = AllowClusteredExpiry; //in case of replication and por, only the coordinator/sub-coordinator is responsible to expire the items. if (!allowExpire) { return(false); } ClusteredArrayList selectedKeys = new ClusteredArrayList(); ClusteredArrayList dependencyChangedSelectedKeys = new ClusteredArrayList(); int oldItemsCount = 0; HashVector oldeItems = null; try { StartLogging(); DateTime startTime = DateTime.Now; int currentTime = AppUtil.DiffSeconds(startTime); if (_context.IsDbSyncCoordinator) { _cdbSyncMgr.AcquireSyncData(); // get the modified keys. and bulk remove them all. this returns // keys from both sql and oledb providers. IDictionary dbkeys = _cdbSyncMgr.GetExpiredKeys(); if (dbkeys.Count > 0) { ClusteredArrayList expire = dbkeys["expire-items"] as ClusteredArrayList; ClusteredArrayList resync = dbkeys["resync-items"] as ClusteredArrayList; if (_context.CacheImpl == null) { return(false); } if (expire.Count > 0) { OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); operationContext.Add(OperationContextFieldName.RaiseCQNotification, true); _topLevelCache.CascadedRemove(expire, ItemRemoveReason.DependencyChanged, true, operationContext); _cdbSyncMgr.FlushSyncData(); //set the flag that item has expired from cache. expired = true; } if (resync.Count > 0 && _context.DsMgr != null) { IEnumerator e = resync.GetEnumerator(); while (e.MoveNext()) { CacheEntry oldEntry = cache.Get(e.Current, new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation)); if (oldEntry != null) { _context.DsMgr.ResyncCacheItemAsync(e.Current, oldEntry.ExpirationHint, null, oldEntry.GroupInfo, oldEntry.QueryInfo, oldEntry.ResyncProviderName); } else { _context.DsMgr.ResyncCacheItemAsync(e.Current, oldEntry.ExpirationHint, null, null, null, oldEntry.ResyncProviderName); } } } } } int cleanSize = (int)Math.Ceiling(cache.Count * _cleanRatio); //set the flag that we are going to expire the items. if (_cacheLastAccessLoggingIntervalPassed >= _cacheLastAccessLoggingInterval) { _cacheLastAccessLoggingInterval = CacheLastAccessLoggingInterval; _cacheLastAccessCountEnabled = IsCacheLastAccessCountEnabled; _cacheLastAccessCountLoggingEnabled = IsCacheLastAccessLoggingEnabled; _cacheLastAccessInterval = CacheLastAccessCountInterval; } else { _cacheLastAccessLoggingIntervalPassed++; } if (_cacheLastAccessCountEnabled && _cacheLastAccessCountLoggingEnabled) { if (_cacheLastAccessLoggingIntervalPassed >= _cacheLastAccessLoggingInterval) { _cacheLastAccessLoggingIntervalPassed = 0; oldeItems = new HashVector(); } } lock (_mainIndex.SyncRoot) { IDictionaryEnumerator em = _mainIndex.GetEnumerator(); //added by muds if (em != null) { while (em.MoveNext()) { ExpirationHint hint = em.Value as ExpirationHint; if (hint != null && _cacheLastAccessCountEnabled && hint is IdleExpiration) { IdleExpiration slidingExpHint = hint as IdleExpiration; TimeSpan diff = AppUtil.GetDateTime(AppUtil.DiffSeconds(DateTime.Now)) - AppUtil.GetDateTime(slidingExpHint.LastAccessTime); if (diff.TotalMinutes >= _cacheLastAccessInterval) { oldItemsCount++; if (oldeItems != null) { oldeItems.Add(em.Key, null); } } } if (hint == null || hint.SortKey.CompareTo(currentTime) >= 0) { continue; } if (!allowExpire && hint.IsRoutable) { continue; } if (hint.DetermineExpiration(_context)) { if (hint.NeedsReSync && _context.DsMgr != null) { //get old entry to know existing groupinfo and queryinfo for tag purposes. CacheEntry oldEntry = cache.Get(em.Key, new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation)); if (oldEntry != null) { _context.DsMgr.ResyncCacheItemAsync(em.Key, hint, null, oldEntry.GroupInfo, oldEntry.QueryInfo, oldEntry.ResyncProviderName); } else { _context.DsMgr.ResyncCacheItemAsync(em.Key, hint, null, null, null, oldEntry.ResyncProviderName); } } else { if (hint.GetExpiringHint() is FixedExpiration || hint.GetExpiringHint() is IdleExpiration) { selectedKeys.Add(em.Key); } else { dependencyChangedSelectedKeys.Add(em.Key); } } if (cleanSize > 0 && selectedKeys.Count == cleanSize) { break; } } } } } if (NCacheLog.IsInfoEnabled) { NCacheLog.Info("ExpirationManager.Expire()", String.Format("Expiry time for {0}/{1} Items: " + (DateTime.UtcNow - startTime), selectedKeys.Count, /*_expiryIndex.KeyCount*/ cache.Count)); } } catch (Exception e) { NCacheLog.Error("ExpirationManager.Expire(bool)", "LocalCache(Expire): " + e.ToString()); } finally { _context.PerfStatsColl.IncrementCacheLastAccessCountStats(oldItemsCount); ApplyLoggs(); ClusteredArrayList dependentItems = new ClusteredArrayList(); ClusteredArrayList removedItems = new ClusteredArrayList(); DateTime startTime = DateTime.Now; HashVector expiredItemTable = new HashVector(); expiredItemTable.Add(ItemRemoveReason.Expired, selectedKeys); //Time based expiration expiredItemTable.Add(ItemRemoveReason.DependencyChanged, dependencyChangedSelectedKeys); //FileDependency or any other try { IDictionaryEnumerator ide = expiredItemTable.GetEnumerator(); while (ide.MoveNext()) { selectedKeys = ide.Value as ClusteredArrayList; ItemRemoveReason removedReason = (ItemRemoveReason)ide.Key; if (selectedKeys.Count > 0) { //new architectural changes begins from here. ClusteredArrayList keysTobeRemoved = new ClusteredArrayList(); for (int i = 0; i < selectedKeys.Count && !_cacheCleared; i++) { keysTobeRemoved.Add(selectedKeys[i]); if (keysTobeRemoved.Count % _removeThreshhold == 0) { try { if (this.IsDisposed) { break; } OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); operationContext.Add(OperationContextFieldName.RaiseCQNotification, true); object[][] keysExposed = keysTobeRemoved.ToInternalArray(); foreach (object[] keyCollection in keysExposed) { ArrayList removed = cache.RemoveSync(keyCollection, removedReason, false, operationContext) as ArrayList; if (removed != null) { removedItems.AddRange(removed); } } //set the flag that item has expired from cache... expired = true; if (_context.PerfStatsColl != null) { _context.PerfStatsColl.IncrementExpiryPerSecStatsBy(keysTobeRemoved.Count); } } catch (Exception e) { NCacheLog.Error("ExpiryManager.Expire", "an error occurred while removing expired items. Error " + e.ToString()); } keysTobeRemoved.Clear(); if (removedItems != null && removedItems.Count > 0) { dependentItems.AddRange(removedItems); removedItems.Clear(); } //we stop the activity of the current thread so that normal user operation is not affected. Thread.Sleep(_sleepInterval * 1000); } } if (!this.IsDisposed && keysTobeRemoved.Count > 0) { try { OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); operationContext.Add(OperationContextFieldName.RaiseCQNotification, true); object[][] keysExposed = keysTobeRemoved.ToInternalArray(); foreach (object[] keyCollection in keysExposed) { ArrayList removed = cache.RemoveSync(keyCollection, removedReason, false, operationContext) as ArrayList; if (removed != null) { removedItems.AddRange(removed); } } //set the flag that item has expired from cache... expired = true; if (_context.PerfStatsColl != null) { _context.PerfStatsColl.IncrementExpiryPerSecStatsBy(keysTobeRemoved.Count); } if (removedItems != null && removedItems.Count > 0) { dependentItems.AddRange(removedItems); removedItems.Clear(); } } catch (Exception e) { NCacheLog.Error("ExpiryManager.Expire", "an error occurred while removing expired items. Error " + e.ToString()); } } } } if (!this.IsDisposed && dependentItems.Count > 0) { ClusteredArrayList removableList = new ClusteredArrayList(); if (rootCache != null) { foreach (object depenentItme in dependentItems) { if (depenentItme == null) { continue; } removableList.Add(depenentItme); if (removableList.Count % 100 == 0) { try { if (this.IsDisposed) { break; } OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); operationContext.Add(OperationContextFieldName.RaiseCQNotification, true); rootCache.CascadedRemove(removableList, ItemRemoveReason.DependencyChanged, true, operationContext); if (_context.PerfStatsColl != null) { _context.PerfStatsColl.IncrementExpiryPerSecStatsBy(removableList.Count); } } catch (Exception exc) { NCacheLog.Error("ExpiryManager.Expire", "an error occurred while removing dependent items. Error " + exc.ToString()); } removableList.Clear(); } } if (!this.IsDisposed && removableList.Count > 0) { try { OperationContext operationContext = new OperationContext(OperationContextFieldName.OperationType, OperationContextOperationType.CacheOperation); operationContext.Add(OperationContextFieldName.RaiseCQNotification, true); rootCache.CascadedRemove(removableList, ItemRemoveReason.DependencyChanged, true, operationContext); if (_context.PerfStatsColl != null) { _context.PerfStatsColl.IncrementExpiryPerSecStatsBy(removableList.Count); } } catch (Exception exc) { NCacheLog.Error("ExpiryManager.Expire", "an error occurred while removing dependent items. Error " + exc.ToString()); } removableList.Clear(); } } } } finally { _transitoryIndex.Clear(); lock (this) { _cacheCleared = false; } if (oldeItems != null) { StringBuilder sb = new StringBuilder(); IDictionaryEnumerator ide = oldeItems.GetEnumerator(); int count = 1; while (ide.MoveNext()) { sb.Append(ide.Key + ", "); if (count % 10 == 0) { sb.Append("\r\n"); count = 1; } else { count++; } } NCacheLog.Info(sb.ToString().Trim()); } } } return(expired); }