/// <summary>
/// Processes the specified query.
/// </summary>
/// <param name="query">The query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>Query Result</returns>
internal TQueryResult Process(BaseMultiIndexIdQuery <TQueryResult> query,
MessageContext messageContext,
IndexStoreContext storeContext)
{
TQueryResult result;
List <ResultItem> resultItemList = new List <ResultItem>();
Dictionary <byte[], IndexHeader> indexIdIndexHeaderMapping = null;
bool isTagPrimarySort = false;
string sortFieldName = null;
List <SortOrder> sortOrderList = null;
int totalCount = 0;
int additionalAvailableItemCount = 0;
GroupByResult groupByResult = null;
StringBuilder exceptionInfo = new StringBuilder();
int indexCap = 0;
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
try
{
if (query.IndexIdList.Count > 0)
{
#region Validate Query
ValidateQuery(indexTypeMapping, query, messageContext);
#endregion
#region Set sort vars
Index targetIndexInfo = indexTypeMapping.IndexCollection[query.TargetIndexName];
indexCap = targetIndexInfo.MaxIndexSize;
if (query.TagSort != null)
{
isTagPrimarySort = query.TagSort.IsTag;
sortFieldName = query.TagSort.TagName;
sortOrderList = new List <SortOrder>(1)
{
query.TagSort.SortOrder
};
}
else
{
isTagPrimarySort = targetIndexInfo.PrimarySortInfo.IsTag;
sortFieldName = targetIndexInfo.PrimarySortInfo.FieldName;
sortOrderList = targetIndexInfo.PrimarySortInfo.SortOrderList;
}
BaseComparer baseComparer = new BaseComparer(isTagPrimarySort, sortFieldName, sortOrderList);
groupByResult = new GroupByResult(baseComparer);
#endregion
#region Prepare ResultList
CacheIndexInternal targetIndex;
IndexIdParams indexIdParam;
int maxExtractCount;
byte[] metadata;
MetadataPropertyCollection metadataPropertyCollection;
Dictionary <KeyValuePair <byte[], string>, CacheIndexInternal> internalIndexDictionary = new Dictionary <KeyValuePair <byte[], string>, CacheIndexInternal>();
int maxMergeCount = query.MaxMergeCount;
IndexCondition queryIndexCondition = query.IndexCondition;
for (int i = 0; i < query.IndexIdList.Count; i++)
{
#region Extract index and apply criteria
indexIdParam = query.GetParamsForIndexId(query.IndexIdList[i]);
maxExtractCount = ComputeMaxExtractCount(indexIdParam.MaxItems,
query.GetAdditionalAvailableItemCount,
indexIdParam.Filter,
query.MaxMergeCount);
// Note: This should be changed later and just extracted once if it is also requested in GetIndexHeader
metadata = null;
metadataPropertyCollection = null;
if (indexTypeMapping.MetadataStoredSeperately)
{
IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping,
messageContext.TypeId,
messageContext.PrimaryId,
query.IndexIdList[i],
storeContext,
out metadata,
out metadataPropertyCollection);
}
targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
(query.PrimaryIdList != null && i < query.PrimaryIdList.Count) ?
query.PrimaryIdList[i] :
IndexCacheUtils.GeneratePrimaryId(query.IndexIdList[i]),
query.IndexIdList[i],
targetIndexInfo.ExtendedIdSuffix,
query.TargetIndexName,
maxExtractCount,
indexIdParam.Filter,
true,
queryIndexCondition,
false,
false,
targetIndexInfo.PrimarySortInfo,
targetIndexInfo.LocalIdentityTagList,
targetIndexInfo.StringHashCodeDictionary,
query.CapCondition,
targetIndexInfo.IsMetadataPropertyCollection,
metadataPropertyCollection,
query.DomainSpecificProcessingType,
storeContext.DomainSpecificConfig,
null,
query.GroupBy,
false);
#endregion
if (targetIndex != null)
{
totalCount += targetIndex.OutDeserializationContext.TotalCount;
additionalAvailableItemCount += targetIndex.Count;
internalIndexDictionary.Add(new KeyValuePair <byte[], string>(query.IndexIdList[i], query.TargetIndexName),
targetIndex);
SetItemCounter(messageContext.TypeId, targetIndex.OutDeserializationContext);
#region Dynamic tag sort
if (query.TagSort != null)
{
targetIndex.Sort(query.TagSort);
}
#endregion
#region Get items from index and merge
if (query.GroupBy == null)
{
MergeAlgo.MergeItemLists(ref resultItemList,
CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, int.MaxValue),
query.MaxMergeCount,
baseComparer);
}
else
{
MergeAlgo.MergeGroupResult(ref groupByResult,
targetIndex.GroupByResult,
query.MaxMergeCount,
baseComparer);
}
if ((i != query.IndexIdList.Count - 1) && (resultItemList.Count == maxMergeCount))
{
AdjustIndexCondition(GetConditionBoundaryBytes(resultItemList, groupByResult, query.GroupBy, isTagPrimarySort, sortFieldName),
ref queryIndexCondition,
baseComparer);
}
#endregion
}
}
#endregion
#region Subset Processing
ProcessSubsets(query, ref resultItemList, ref groupByResult, baseComparer);
#endregion
#region Get Extra Tags for IndexIds in the list
//Note: Getting extra tags from GroupByResult not supported for now
if (query.TagsFromIndexes != null && query.TagsFromIndexes.Count != 0)
{
KeyValuePair <byte[] /*IndexId */, string /*IndexName*/> kvp;
CacheIndexInternal additionalCacheIndexInternal;
#region Form IndexId - PrimaryId Mapping
Dictionary <byte[] /*IndexId */, int /*PrimaryId*/> indexIdPrimaryIdMapping =
new Dictionary <byte[] /*IndexId */, int /*PrimaryId*/>(query.IndexIdList.Count, new ByteArrayEqualityComparer());
if (query.PrimaryIdList != null && query.PrimaryIdList.Count > 0)
{
//Form dictionary of IndexIdPrimaryIdMapping
for (int i = 0; i < query.IndexIdList.Count && i < query.PrimaryIdList.Count; i++)
{
indexIdPrimaryIdMapping.Add(query.IndexIdList[i], query.PrimaryIdList[i]);
}
}
#endregion
int indexPrimaryId;
foreach (ResultItem resultItem in resultItemList)
{
foreach (string indexName in query.TagsFromIndexes)
{
Index indexInfo = indexTypeMapping.IndexCollection[indexName];
kvp = new KeyValuePair <byte[], string>(resultItem.IndexId, indexName);
if (!internalIndexDictionary.ContainsKey(kvp))
{
additionalCacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
indexIdPrimaryIdMapping.TryGetValue(resultItem.IndexId, out indexPrimaryId) ?
indexPrimaryId :
IndexCacheUtils.GeneratePrimaryId(resultItem.IndexId),
resultItem.IndexId,
indexInfo.ExtendedIdSuffix,
indexName,
0,
null,
true,
null,
false,
false,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null,
indexInfo.IsMetadataPropertyCollection,
null,
query.DomainSpecificProcessingType,
storeContext.DomainSpecificConfig,
null,
null,
false);
if (additionalCacheIndexInternal != null)
{
SetItemCounter(messageContext.TypeId, additionalCacheIndexInternal.OutDeserializationContext);
internalIndexDictionary.Add(kvp, additionalCacheIndexInternal);
try
{
IndexServerUtils.GetTags(additionalCacheIndexInternal, resultItem, resultItem);
}
catch (Exception ex)
{
LoggingUtil.Log.Error(ex.ToString());
exceptionInfo.Append(" | " + ex.Message);
}
}
}
}
}
}
#endregion
#region Get IndexHeader
if (query.GetIndexHeaderType == GetIndexHeaderType.AllIndexIds)
{
//Get IndexHeader for all IndexIds
indexIdIndexHeaderMapping = new Dictionary <byte[], IndexHeader>(new ByteArrayEqualityComparer());
for (int i = 0; i < query.IndexIdList.Count; i++)
{
byte[] indexId = query.IndexIdList[i];
CacheIndexInternal targetIndexCacheIndexInternal;
if (!indexIdIndexHeaderMapping.ContainsKey(indexId) &&
internalIndexDictionary.TryGetValue(new KeyValuePair <byte[], string>(indexId, query.TargetIndexName), out targetIndexCacheIndexInternal))
{
indexIdIndexHeaderMapping.Add(indexId, GetIndexHeader(internalIndexDictionary,
targetIndexCacheIndexInternal,
indexId,
query,
indexTypeMapping,
messageContext.TypeId,
storeContext,
i));
}
}
}
else if (query.GetIndexHeaderType == GetIndexHeaderType.ResultItemsIndexIds)
{
//Get IndexHeader just for IndexIds present in the result
indexIdIndexHeaderMapping = new Dictionary <byte[], IndexHeader>(new ByteArrayEqualityComparer());
if (query.GroupBy == null)
{
for (int i = 0; i < resultItemList.Count; i++)
{
ResultItem resultItem = resultItemList[i];
if (!indexIdIndexHeaderMapping.ContainsKey(resultItem.IndexId))
{
CacheIndexInternal targetIndexCacheIndexInternal;
internalIndexDictionary.TryGetValue(new KeyValuePair <byte[], string>(resultItem.IndexId, query.TargetIndexName),
out targetIndexCacheIndexInternal);
indexIdIndexHeaderMapping.Add(resultItem.IndexId,
GetIndexHeader(internalIndexDictionary,
targetIndexCacheIndexInternal,
resultItem.IndexId,
query,
indexTypeMapping,
messageContext.TypeId,
storeContext,
i));
}
}
}
else
{
foreach (ResultItemBag resultItemBag in groupByResult)
{
for (int i = 0; i < resultItemBag.Count; i++)
{
ResultItem resultItem = resultItemBag[i];
if (!indexIdIndexHeaderMapping.ContainsKey(resultItem.IndexId))
{
CacheIndexInternal targetIndexCacheIndexInternal;
internalIndexDictionary.TryGetValue(new KeyValuePair <byte[], string>(resultItem.IndexId, query.TargetIndexName),
out targetIndexCacheIndexInternal);
indexIdIndexHeaderMapping.Add(resultItem.IndexId,
GetIndexHeader(internalIndexDictionary,
targetIndexCacheIndexInternal,
resultItem.IndexId,
query,
indexTypeMapping,
messageContext.TypeId,
storeContext,
i));
}
}
}
}
}
#endregion
#region Get data
if (!query.ExcludeData)
{
DataTierUtil.GetData(resultItemList,
groupByResult,
storeContext, messageContext,
indexTypeMapping.FullDataIdFieldList,
query.FullDataIdInfo);
}
#endregion
}
result = new TQueryResult
{
ResultItemList = resultItemList,
IndexIdIndexHeaderMapping = indexIdIndexHeaderMapping,
TotalCount = totalCount,
AdditionalAvailableItemCount = additionalAvailableItemCount,
IsTagPrimarySort = isTagPrimarySort,
SortFieldName = sortFieldName,
SortOrderList = sortOrderList,
IndexCap = indexCap,
GroupByResult = groupByResult,
ExceptionInfo = exceptionInfo.ToString()
};
#region Log Potentially Bad Queries
if (indexTypeMapping.QueryOverrideSettings != null &&
indexTypeMapping.QueryOverrideSettings.MaxResultItemsThresholdLog > 0 &&
resultItemList != null &&
resultItemList.Count > indexTypeMapping.QueryOverrideSettings.MaxResultItemsThresholdLog)
{
LoggingUtil.Log.ErrorFormat("Encountered potentially Bad Paged Query with Large Result Set of {0}. AddressHistory: {1}. Query Info: {2}",
resultItemList.Count,
IndexServerUtils.FormatAddressHistory(messageContext.AddressHistory),
FormatQueryInfo(query));
}
LoggingUtil.Log.DebugFormat("QueryInfo: {0}, AddressHistory: {1}", FormatQueryInfo(query), IndexServerUtils.FormatAddressHistory(messageContext.AddressHistory));
#endregion
SetIndexIdListCounter(messageContext.TypeId, query);
}
catch (Exception ex)
{
exceptionInfo.Append(" | " + ex.Message);
result = new TQueryResult
{
ExceptionInfo = exceptionInfo.ToString()
};
LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing PagedIndexQuery : {1}", messageContext.TypeId, ex);
}
return(result);
}
/// <summary>
/// Processes the specified contains index query.
/// </summary>
/// <param name="containsIndexQuery">The contains index query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>ContainsIndexQueryResult</returns>
internal static ContainsIndexQueryResult Process(ContainsIndexQuery containsIndexQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
ContainsIndexQueryResult containsIndexQueryResult;
MultiItemResult multiItemResult = null;
byte[] metadata = null;
MetadataPropertyCollection metadataPropertyCollection = null;
bool indexExists = false;
int indexSize = -1;
int virtualCount = -1;
try
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
#region Check TargetIndexName
if (string.IsNullOrEmpty(containsIndexQuery.TargetIndexName))
{
containsIndexQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping);
}
if (!indexTypeMapping.IndexCollection.Contains(containsIndexQuery.TargetIndexName))
{
throw new Exception("Invalid TargetIndexName - " + containsIndexQuery.TargetIndexName);
}
#endregion
Index targetIndexInfo = indexTypeMapping.IndexCollection[containsIndexQuery.TargetIndexName];
int indexCap = targetIndexInfo.MaxIndexSize;
List <CacheIndexInternal> internalCacheIndexList = new List <CacheIndexInternal>();
#region Get TargetIndex
CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
containsIndexQuery.IndexId,
targetIndexInfo.ExtendedIdSuffix,
containsIndexQuery.TargetIndexName,
0,
null,
true,
null,
false,
false,
targetIndexInfo.PrimarySortInfo,
targetIndexInfo.LocalIdentityTagList,
targetIndexInfo.StringHashCodeDictionary,
null,
targetIndexInfo.IsMetadataPropertyCollection,
null,
containsIndexQuery.DomainSpecificProcessingType,
storeContext.DomainSpecificConfig,
null,
null,
false);
#endregion
if (cacheIndexInternal != null)
{
internalCacheIndexList.Add(cacheIndexInternal);
indexExists = true;
indexSize = cacheIndexInternal.OutDeserializationContext.TotalCount;
virtualCount = cacheIndexInternal.VirtualCount;
// update the performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery,
messageContext.TypeId,
indexSize);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery,
messageContext.TypeId,
cacheIndexInternal.OutDeserializationContext.ReadItemCount);
int searchIndex;
IndexDataItem indexDataItem;
foreach (IndexItem queryIndexItem in containsIndexQuery.IndexItemList)
{
#region Search item in index
searchIndex = internalCacheIndexList[0].Search(queryIndexItem);
#endregion
if (searchIndex > -1)
{
if (multiItemResult == null)
{
multiItemResult = new MultiItemResult(containsIndexQuery.IndexId);
}
indexDataItem = new IndexDataItem(InternalItemAdapter.ConvertToIndexItem(internalCacheIndexList[0].GetItem(searchIndex),
internalCacheIndexList[0].InDeserializationContext));
#region Get extra tags
if (containsIndexQuery.TagsFromIndexes != null && containsIndexQuery.TagsFromIndexes.Count != 0)
{
foreach (string indexName in containsIndexQuery.TagsFromIndexes)
{
Index indexInfo = indexTypeMapping.IndexCollection[indexName];
CacheIndexInternal indexInternal =
IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
containsIndexQuery.IndexId,
indexInfo.ExtendedIdSuffix,
indexName,
0,
null,
true,
null,
false,
false,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null,
indexInfo.IsMetadataPropertyCollection,
null,
containsIndexQuery.DomainSpecificProcessingType,
storeContext.DomainSpecificConfig,
null,
null,
false);
if (indexInternal != null)
{
// update the performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery,
messageContext.TypeId,
indexInternal.OutDeserializationContext.TotalCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery,
messageContext.TypeId,
indexInternal.OutDeserializationContext.ReadItemCount);
internalCacheIndexList.Add(indexInternal);
IndexServerUtils.GetTags(indexInternal, queryIndexItem, indexDataItem);
}
}
}
#endregion
multiItemResult.Add(indexDataItem);
}
}
#region Get data
if (!containsIndexQuery.ExcludeData && multiItemResult != null)
{
byte[] extendedId;
List <RelayMessage> dataStoreMessages = new List <RelayMessage>(multiItemResult.Count);
short relatedTypeId;
if (containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null)
{
if (!storeContext.TryGetTypeId(containsIndexQuery.FullDataIdInfo.RelatedTypeName, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", containsIndexQuery.FullDataIdInfo.RelatedTypeName);
throw new Exception("Invalid RelatedTypeId for TypeId - " + containsIndexQuery.FullDataIdInfo.RelatedTypeName);
}
}
else if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
foreach (IndexDataItem resultItem in multiItemResult)
{
extendedId = DataTierUtil.GetFullDataId(containsIndexQuery.IndexId,
resultItem,
containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null ?
containsIndexQuery.FullDataIdInfo.FullDataIdFieldList :
indexTypeMapping.FullDataIdFieldList);
dataStoreMessages.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(extendedId), extendedId, MessageType.Get));
}
storeContext.ForwarderComponent.HandleMessages(dataStoreMessages);
int i = 0;
foreach (IndexDataItem resultItem in multiItemResult)
{
if (dataStoreMessages[i].Payload != null)
{
resultItem.Data = dataStoreMessages[i].Payload.ByteArray;
}
i++;
}
}
#endregion
#region Get metadata
if (containsIndexQuery.GetMetadata)
{
if (indexTypeMapping.MetadataStoredSeperately)
{
IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping,
messageContext.TypeId,
messageContext.PrimaryId,
containsIndexQuery.IndexId,
storeContext,
out metadata,
out metadataPropertyCollection);
}
else
{
IndexServerUtils.GetMetadataStoredWithIndex(indexTypeMapping,
internalCacheIndexList,
out metadata,
out metadataPropertyCollection);
}
}
#endregion
}
containsIndexQueryResult = new ContainsIndexQueryResult(multiItemResult,
metadata,
metadataPropertyCollection,
indexSize,
indexExists,
virtualCount,
indexCap,
null);
}
catch (Exception ex)
{
containsIndexQueryResult = new ContainsIndexQueryResult(null, null, null, -1, false, -1, 0, ex.Message);
LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing ContainsIndexQuery : {1}", messageContext.TypeId, ex);
}
return(containsIndexQueryResult);
}