private static object BuildItemListWithSelect(DataQueryResults results, Type itemType, Expression selectExpression)
{
var itemList = (IList)Activator.CreateInstance(typeof(List <>).MakeGenericType(itemType));
if (itemList == null)
{
return(null);
}
var converter = new SelectExpressionConverter(results.Fieldset);
foreach (var dataItem in results.RecordList)
{
// I take the original select expression and then substitute the member expressions for constant data values
//var newExpression = converter.Convert(selectExpression, dataItem.ItemArray.Take(dataItem.ItemArray.Length - 2).ToArray());
var newExpression = converter.Convert(selectExpression, dataItem.ItemArray);
// Magic happens here.
// We then compile and execute the expression which create a new object that has been properly initialized
var objectMember = Expression.Convert(newExpression, typeof(object));
var getterLambda = Expression.Lambda <Func <object> >(objectMember);
var getter = getterLambda.Compile();
var obj = getter();
itemList.Add(obj);
}
return(itemList);
}
private static object BuildItemListWithSelect2(DataQueryResults results, Type itemType, Expression selectExpression)
{
var itemList = (IList)Activator.CreateInstance(typeof(List <>).MakeGenericType(itemType));
if (itemList == null)
{
return(null);
}
foreach (var record in results.RecordList)
{
//Load all derived fields first
var objArray = new object[results.Query.FieldSelects.Count];
var index = 0;
foreach (var fieldName in results.Query.FieldSelects)
{
if (fieldName == "Key")
{
objArray[index] = record.ItemArray[0];
}
else if (results.Query.DerivedFieldList.Any(x => x.Alias == fieldName))
{
var dfIndex = results.Query.DerivedFieldList.IndexOf(x => x.Alias == fieldName) + results.Query.GroupFields.Count;
objArray[index] = record.ItemArray[dfIndex];
}
index++;
}
//Create the actual object
var obj = Activator.CreateInstance(itemType, objArray);
itemList.Add(obj);
}
return(itemList);
}
private static object BuildItemFromDerivedFields(DataQueryResults results, Type itemType)
{
var index = 0;
var constructorParams = new object[results.DerivedFieldList.Length];
foreach (var derivedField in results.DerivedFieldList)
{
constructorParams[index++] = derivedField.Value;
}
return(Activator.CreateInstance(itemType, constructorParams));
}
private static object BuildItemList(DataQueryResults results, Type itemType, Expression selectExpression)
{
if (results.Query.GroupFields?.Count > 0)
{
//GroupBy
return(BuildItemListWithSelect2(results, itemType, selectExpression));
}
else
{
//Normal projection
if (selectExpression == null)
{
return(BuildItemList(results, itemType));
}
return(BuildItemListWithSelect(results, itemType, selectExpression));
}
}
private static object BuildItemFromDerivedFields2(DataQueryResults results, Type itemType)
{
return(results.DerivedFieldList.Any() ? results.DerivedFieldList.First().Value : null);
}
private static object BuildItemList(DataQueryResults results, Type itemType)
{
var itemList = (IList)Activator.CreateInstance(typeof(List <>).MakeGenericType(itemType));
if (itemList == null)
{
return(null);
}
foreach (var dataItem in results.RecordList)
{
var obj = Activator.CreateInstance(itemType);
var dataStoreItem = obj as IDatastoreItem;
if (dataStoreItem != null)
{
var index = 0;
foreach (var fieldset in results.Fieldset)
{
var value = dataItem.ItemArray[index];
var property = itemType.GetProperty(fieldset.Name);
if (property != null && property.CanWrite)
{
if (value == null)
{
property.SetValue(obj, null);
}
else if (property.PropertyType == typeof(Single?) || property.PropertyType == typeof(Single))
{
property.SetValue(obj, Convert.ToSingle(value));
}
else if (property.PropertyType == typeof(decimal?) || property.PropertyType == typeof(decimal))
{
property.SetValue(obj, Convert.ToDecimal(value));
}
else if (property.PropertyType == typeof(byte?) || property.PropertyType == typeof(byte))
{
property.SetValue(obj, Convert.ToByte(value));
}
else if (property.PropertyType == typeof(short?) || property.PropertyType == typeof(short))
{
property.SetValue(obj, Convert.ToInt16(value));
}
else
{
property.SetValue(obj, value);
}
}
else
{
if (dataStoreItem.ExtraValues == null)
{
dataStoreItem.ExtraValues = new Dictionary <string, string>();
}
dataStoreItem.ExtraValues.Add(fieldset.Name, value?.ToString());
}
index++;
}
dataStoreItem.__OrdinalPosition = dataItem.__OrdinalPosition;
dataStoreItem.__RecordIndex = dataItem.__RecordIndex;
dataStoreItem.__Timestamp = dataItem.__Timestamp;
itemList.Add(obj);
}
}
return(itemList);
}
private static void SetResultItems(object resultObj, Type itemType, DataQuery dataQuery, DataQueryResults results, Expression selectExpression)
{
if (dataQuery.RecordsPerPage > 0)
{
var totalPageCount = (results.TotalRecordCount / dataQuery.RecordsPerPage) + (results.TotalRecordCount % dataQuery.RecordsPerPage == 0 ? 0 : 1);
SetProperty(resultObj, "TotalPageCount", totalPageCount);
}
SetProperty(resultObj, "TotalRecordCount", results.TotalRecordCount);
SetProperty(resultObj, "Query", results.Query);
SetProperty(resultObj, "Items", BuildItemList(results, itemType, selectExpression));
}
public void Set(DatastoreEntities context, RepositorySchema schema, DataQuery query, int repositoryId, Guid id, DataQueryResults results)
{
if (!ConfigHelper.AllowCaching)
{
return;
}
if (results == null)
{
return;
}
//Do not cache big items
if (results.RecordList.Count > 100)
{
return;
}
if (!string.IsNullOrEmpty(query.Keyword) && !this.FTSReadyCache.IsReady(id))
{
return;
}
//if (!string.IsNullOrEmpty(query.Keyword) && !ConfigHelper.AllowCacheWithKeyword) return;
var timer = Stopwatch.StartNew();
var cache = RepositoryCacheManager.GetCache(id, RepositoryManager.GetSchemaParentId(repositoryId));
long lockTime = 0;
var changeStamp = 0;
var queryHash = 0;
var subCacheKey = GetSubKey(schema, query);
try
{
//Some queries should be cached a long time
var longCache = !query.FieldFilters.Any() &&
!query.FieldSorts.Any() &&
string.IsNullOrEmpty(query.Keyword) &&
!query.SkipDimensions.Any();
var extraMinutes = longCache ? 480 : 0;
var coreHash = 0;
CacheResultsQuery item;
using (var q = new AcquireReaderLock(ServerUtilities.RandomizeGuid(cache.ID, RSeed), "QueryCache"))
{
lockTime += q.LockTime;
queryHash = query.GetHashCode();
if (!query.ExcludeCount && query.IncludeDimensions && !query.IncludeEmptyDimensions)
{
coreHash = query.CoreHashCode();
}
changeStamp = RepositoryManager.GetRepositoryChangeStamp(context, repositoryId);
lock (cache)
{
item = cache?.FirstOrDefault(x => x.QueryHash == queryHash && x.ChangeStamp == changeStamp);
}
//If data has not changed and results are in cache then do nothing except mark as accessed
if (item != null)
{
item.Results = results;
item.Timestamp = DateTime.Now.AddMinutes(extraMinutes);
item.SubKey = subCacheKey;
return;
}
}
lock (cache)
{
using (var q = new AcquireWriterLock(ServerUtilities.RandomizeGuid(cache.ID, RSeed), "QueryCache"))
{
lockTime += q.LockTime;
//Create a new cache item
item = new CacheResultsQuery()
{
QueryHash = queryHash,
QueryCoreHash = coreHash,
RepositoryId = repositoryId,
ChangeStamp = changeStamp,
Results = results,
QueryString = query.ToString(),
ParentId = RepositoryManager.GetSchemaParentId(repositoryId),
Timestamp = DateTime.Now.AddMinutes(extraMinutes),
SubKey = subCacheKey,
};
cache.Add(item);
}
}
}
catch (Exception ex)
{
timer.Stop();
LoggerCQ.LogError(ex, $"RepositoryId={id}, Elapsed={timer.ElapsedMilliseconds}, ID={id}, LockTime={lockTime}, Count={cache.Count}, QueryHash={queryHash}, ChangeStamp={changeStamp}");
throw;
}
finally
{
timer.Stop();
if (timer.ElapsedMilliseconds > 50)
{
LoggerCQ.LogWarning($"Slow cache set: Elapsed={timer.ElapsedMilliseconds}, LockTime={lockTime}, Count={cache.Count}, ID={id}, Query=\"{query.ToString()}\"");
}
LoggerCQ.LogTrace($"QueryCache: Set: SubCacheKey={subCacheKey}");
}
}
/// <summary>
/// This timer will crawl through the data and find any item with no hash
/// It will then then select records to activate the hash generation routine
/// </summary>
private static void _timerCheck_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
//If the system is turned off then do nothing
if (!ConfigHelper.EnabledDataManager)
{
return;
}
if (!IsActive)
{
return;
}
if (!EnableHouseKeeping)
{
return;
}
_timerCheck.Enabled = false;
try
{
const int BlockCount = 50;
const int WaitTime = 250;
var core = ((SystemCore)RepositoryManager.SystemCore);
while (_highPriority.Any())
{
//Try to get an ID from the high priority list
//if none found then grab an arbitrary one from the global list
Guid ID = Guid.Empty;
if (!_highPriority.TryTake(out ID))
{
return;
}
if (ID == Guid.Empty)
{
return;
}
//If there was an error it is in the skip list so never handle this ID again
if (!_skipList.Any(x => x == ID))
{
var sb = new StringBuilder();
var schema = RepositoryManager.GetSchema(ID);
if (schema != null)
{
DataQueryResults results = new DataQueryResults();
do
{
if (!IsActive)
{
return; //Stop when housekeeping comes on
}
var query = new DataQuery {
IncludeRecords = true, IncludeDimensions = false, ExcludeCount = true
};
query.FieldFilters.Add(new FieldFilter {
Name = SqlHelper.HashField, Comparer = ComparisonConstants.Equals, Value = 0
});
query.RecordsPerPage = BlockCount;
try
{
results = core.Manager.Query(ID, query, true);
Interlocked.Add(ref _counter, results.RecordList.Count);
}
catch
{
results = new DataQueryResults();
}
//Do not overload the system with background queries
if (results.ComputeTime > 2000)
{
System.Threading.Thread.Sleep(WaitTime * 2);
}
else if (core.GetCpu() > 60)
{
System.Threading.Thread.Sleep(WaitTime * 8);
}
else
{
System.Threading.Thread.Sleep(WaitTime);
}
} while (results.RecordList?.Count == BlockCount && results.ComputeTime < 5000);
}
}
}
if (InFullIndex)
{
SystemCore.PatchApply(FullIndexPatch, "DataManager:FullIndex");
}
}
catch (Exception ex)
{
LoggerCQ.LogError(ex);
}
finally
{
_timerCheck.Enabled = true;
}
}
