private void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
List <Gravitybox.Datastore.EFDAL.Entity.RepositoryLog> work;
try
{
//Copy the cache list and empty so other threads can continue to use it
using (var q = new AcquireReaderLock(QueryLogID, "QueryLog"))
{
work = _cache.ToList();
_cache.Clear();
}
//Save all of these to disk
using (var context = new DatastoreEntities(ConfigHelper.ConnectionString))
{
foreach (var item in work)
{
context.AddItem(item);
}
context.SaveChanges();
}
}
catch (Exception ex)
{
LoggerCQ.LogWarning($"QueryLogManager: Error={ex.Message}");
}
}
public void SetSlice(DatastoreEntities context, SummarySlice slice, int repositoryId, SummarySliceValue results)
{
if (!ConfigHelper.AllowCaching)
{
return;
}
int changeStamp = 0;
int queryHash = 0;
CacheResultsSlice item;
using (var q = new AcquireReaderLock(QueryCacheID, "QueryCache"))
{
//Do not cache big items
if (results.RecordList.Count > 500)
{
return;
}
if (slice.Query != null && !string.IsNullOrEmpty(slice.Query.Keyword) && !ConfigHelper.AllowCacheWithKeyword)
{
return;
}
queryHash = slice.GetHashCode();
changeStamp = RepositoryManager.GetRepositoryChangeStamp(context, repositoryId);
item = _cacheSlice.FirstOrDefault(x => x.QueryHash == queryHash &&
x.RepositoryId == repositoryId &&
x.ChangeStamp == changeStamp);
//If data has not changed and results are in cache then do nothing except mark as accessed
if (item != null)
{
item.Results = results;
item.Timestamp = DateTime.Now;
return;
}
}
using (var q = new AcquireWriterLock(QueryCacheID, "QueryCache"))
{
//Remove previous cache
_cacheSlice.RemoveAll(x => x.QueryHash == queryHash && x.RepositoryId == repositoryId);
//Create a new cache item
item = new CacheResultsSlice()
{
QueryHash = queryHash,
RepositoryId = repositoryId,
ChangeStamp = changeStamp,
Results = results,
QueryString = slice.ToString(),
ParentId = RepositoryManager.GetSchemaParentId(repositoryId),
};
_cacheSlice.Add(item);
}
}
public SummarySliceValue GetSlice(DatastoreEntities context, SummarySlice slice, int repositoryId)
{
if (!ConfigHelper.AllowCaching)
{
return(null);
}
using (var q = new AcquireReaderLock(QueryCacheID, "QueryCache"))
{
var queryHash = slice.GetHashCode();
var changeStamp = RepositoryManager.GetRepositoryChangeStamp(context, repositoryId);
var item = _cacheSlice.FirstOrDefault(x => x.QueryHash == queryHash && x.RepositoryId == repositoryId && x.ChangeStamp == changeStamp);
if (item == null)
{
return(null);
}
item.Timestamp = DateTime.Now;
item.HitCount++;
return(item.Results);
}
}
public void Set(DatastoreEntities context, RepositorySchema schema, DataQuery query, int repositoryId, Guid id, DataQueryResults results)
{
if (!ConfigHelper.AllowCaching)
{
return;
}
if (results == null)
{
return;
}
//Do not cache big items
if (results.RecordList.Count > 100)
{
return;
}
if (!string.IsNullOrEmpty(query.Keyword) && !this.FTSReadyCache.IsReady(id))
{
return;
}
//if (!string.IsNullOrEmpty(query.Keyword) && !ConfigHelper.AllowCacheWithKeyword) return;
var timer = Stopwatch.StartNew();
var cache = RepositoryCacheManager.GetCache(id, RepositoryManager.GetSchemaParentId(repositoryId));
long lockTime = 0;
var changeStamp = 0;
var queryHash = 0;
var subCacheKey = GetSubKey(schema, query);
try
{
//Some queries should be cached a long time
var longCache = !query.FieldFilters.Any() &&
!query.FieldSorts.Any() &&
string.IsNullOrEmpty(query.Keyword) &&
!query.SkipDimensions.Any();
var extraMinutes = longCache ? 480 : 0;
var coreHash = 0;
CacheResultsQuery item;
using (var q = new AcquireReaderLock(ServerUtilities.RandomizeGuid(cache.ID, RSeed), "QueryCache"))
{
lockTime += q.LockTime;
queryHash = query.GetHashCode();
if (!query.ExcludeCount && query.IncludeDimensions && !query.IncludeEmptyDimensions)
{
coreHash = query.CoreHashCode();
}
changeStamp = RepositoryManager.GetRepositoryChangeStamp(context, repositoryId);
lock (cache)
{
item = cache?.FirstOrDefault(x => x.QueryHash == queryHash && x.ChangeStamp == changeStamp);
}
//If data has not changed and results are in cache then do nothing except mark as accessed
if (item != null)
{
item.Results = results;
item.Timestamp = DateTime.Now.AddMinutes(extraMinutes);
item.SubKey = subCacheKey;
return;
}
}
lock (cache)
{
using (var q = new AcquireWriterLock(ServerUtilities.RandomizeGuid(cache.ID, RSeed), "QueryCache"))
{
lockTime += q.LockTime;
//Create a new cache item
item = new CacheResultsQuery()
{
QueryHash = queryHash,
QueryCoreHash = coreHash,
RepositoryId = repositoryId,
ChangeStamp = changeStamp,
Results = results,
QueryString = query.ToString(),
ParentId = RepositoryManager.GetSchemaParentId(repositoryId),
Timestamp = DateTime.Now.AddMinutes(extraMinutes),
SubKey = subCacheKey,
};
cache.Add(item);
}
}
}
catch (Exception ex)
{
timer.Stop();
LoggerCQ.LogError(ex, $"RepositoryId={id}, Elapsed={timer.ElapsedMilliseconds}, ID={id}, LockTime={lockTime}, Count={cache.Count}, QueryHash={queryHash}, ChangeStamp={changeStamp}");
throw;
}
finally
{
timer.Stop();
if (timer.ElapsedMilliseconds > 50)
{
LoggerCQ.LogWarning($"Slow cache set: Elapsed={timer.ElapsedMilliseconds}, LockTime={lockTime}, Count={cache.Count}, ID={id}, Query=\"{query.ToString()}\"");
}
LoggerCQ.LogTrace($"QueryCache: Set: SubCacheKey={subCacheKey}");
}
}
public DataQueryResults Get(DatastoreEntities context, RepositorySchema schema, DataQuery query, int repositoryId, Guid id, out bool isCore)
{
isCore = false;
if (!ConfigHelper.AllowCaching)
{
return(null);
}
long lockTime = 0;
int queryHash = 0;
int coreHash = 0;
int changeStamp = 0;
var task1 = Task.Factory.StartNew(() =>
{
queryHash = query.GetHashCode();
coreHash = query.CoreHashCode();
changeStamp = RepositoryManager.GetRepositoryChangeStamp(context, repositoryId);
});
var timer = new Stopwatch();
var cache = RepositoryCacheManager.GetCache(id, RepositoryManager.GetSchemaParentId(repositoryId));
try
{
using (var q = new AcquireReaderLock(ServerUtilities.RandomizeGuid(cache.ID, RSeed), "QueryCache"))
{
lockTime = q.LockTime;
timer.Start();
//Ensure that the pre-calculations are complete
task1.Wait();
CacheResultsQuery item = null;
lock (cache)
{
item = cache?.FirstOrDefault(x => x.QueryHash == queryHash && x.ChangeStamp == changeStamp);
}
if (item == null) //return null;
{
if (ConfigHelper.AllowCoreCache)
{
//TODO: OPTIMIZE: this is a linear search of thousands of items!!!!
//If did not find a match then find see if core properties match
//If so we can use the dimension and count values and just replace the records collection
lock (cache)
{
item = cache?.FirstOrDefault(x => x.QueryCoreHash == coreHash && x.ChangeStamp == changeStamp);
}
}
if (item == null)
{
return(null);
}
isCore = true;
item.HitCount++;
return(item.Results);
}
item.Timestamp = DateTime.Now;
item.HitCount++;
return(item.Results);
}
}
catch (Exception ex)
{
timer.Stop();
LoggerCQ.LogError(ex, $"RepositoryId={id}, Elapsed={timer.ElapsedMilliseconds}, LockTime={lockTime}, Count={cache.Count}, QueryHash={queryHash}, ChangeStamp={changeStamp}, ID={id}");
throw;
}
finally
{
timer.Stop();
if (timer.ElapsedMilliseconds > 50)
{
LoggerCQ.LogWarning($"Slow cache get: Elapsed={timer.ElapsedMilliseconds}, LockTime={lockTime}, Count={cache.Count}, ID={id}, QueryString=\"{query.ToString()}\"");
}
}
}
