private static void ValidateQuery(MultiIndexContainsQuery multiIndexContainsQuery, IndexTypeMapping indexTypeMapping, IndexStoreContext storeContext, short typeId, out short relatedTypeId) { relatedTypeId = -1; if (string.IsNullOrEmpty(multiIndexContainsQuery.TargetIndexName)) { multiIndexContainsQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(multiIndexContainsQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + multiIndexContainsQuery.TargetIndexName); } if (!multiIndexContainsQuery.ExcludeData) { if (multiIndexContainsQuery.FullDataIdInfo != null && multiIndexContainsQuery.FullDataIdInfo.RelatedTypeName != null) { if (!storeContext.TryGetTypeId(multiIndexContainsQuery.FullDataIdInfo.RelatedTypeName, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", multiIndexContainsQuery.FullDataIdInfo.RelatedTypeName); throw new Exception("Invalid RelatedTypeId for TypeId - " + multiIndexContainsQuery.FullDataIdInfo.RelatedTypeName); } } else if (!storeContext.TryGetRelatedIndexTypeId(typeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", typeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + typeId); } } }
/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>Raw CacheIndexInternal</returns> internal static byte[] Process(MessageContext messageContext, IndexStoreContext storeContext) { byte[] payloadByteArray = null; RelayMessage getMsg = new RelayMessage { MessageType = MessageType.Get, Id = IndexCacheUtils.GeneratePrimaryId(messageContext.ExtendedId), ExtendedId = IndexServerUtils.FormExtendedId(messageContext.ExtendedId, 0) // Pull first of the multiple indexes }; if (storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection.Contains(messageContext.TypeId)) { getMsg.TypeId = messageContext.TypeId; } else if (storeContext.RelatedTypeIds.TryGetValue(messageContext.TypeId, out getMsg.TypeId)) { } else { LoggingUtil.Log.InfoFormat("Invalid TypeID for GetMessage {0}", messageContext.TypeId); return(payloadByteArray); } storeContext.IndexStorageComponent.HandleMessage(getMsg); if (getMsg.Payload != null) { payloadByteArray = getMsg.Payload.ByteArray; } return(payloadByteArray); }
/// <summary> /// Validates the query. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="intersectionQuery">The intersection query.</param> private static void ValidateQuery(IndexTypeMapping indexTypeMapping, IntersectionQuery intersectionQuery) { if (intersectionQuery.IsSingleClusterQuery && intersectionQuery.IndexIdList.Count == 1) { throw new Exception("Intersection requires minimum two Index Ids in the IndexIdList"); } if (string.IsNullOrEmpty(intersectionQuery.TargetIndexName)) { intersectionQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(intersectionQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + intersectionQuery.TargetIndexName); } if (intersectionQuery.PrimaryIdList != null && intersectionQuery.PrimaryIdList.Count != intersectionQuery.IndexIdList.Count) { throw new Exception("PrimaryIdList.Count does not match with IndexIdList.Count"); } if (!intersectionQuery.ExcludeData && FullDataIdContainsIndexId(indexTypeMapping.FullDataIdFieldList)) { throw new Exception("IntersectionQuery.ExcludeData cannot be set to true since FullDataId contains IndexId"); } }
/// <summary> /// Performs the query override. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="query">The query.</param> /// <param name="messageContext">The message context.</param> private void PerformQueryOverride(IndexTypeMapping indexTypeMapping, BaseMultiIndexIdQuery <PagedIndexQueryResult> query, MessageContext messageContext) { PagedIndexQuery pagedQuery = query as PagedIndexQuery; if (indexTypeMapping.QueryOverrideSettings != null) { // override MaxItemsPerIndexThreshold if required if (indexTypeMapping.QueryOverrideSettings.MaxItemsPerIndexThreshold != 0 && pagedQuery.IndexIdList.Count > 1 && pagedQuery.PageNum == 0 && (pagedQuery.IndexIdParamsMapping == null || pagedQuery.IndexIdParamsMapping.Count == 0) && pagedQuery.MaxItemsPerIndex == 0 && pagedQuery.Filter == null) { LoggingUtil.Log.ErrorFormat( "Encountered Potentially Bad Paged Query. Overriding MaxItemsPerIndex to {0}. AddressHistory {1}. Original Query Info: {2}", indexTypeMapping.QueryOverrideSettings.MaxItemsPerIndexThreshold, IndexServerUtils.FormatAddressHistory(messageContext.AddressHistory), FormatQueryInfo(pagedQuery)); pagedQuery.MaxItemsPerIndex = indexTypeMapping.QueryOverrideSettings.MaxItemsPerIndexThreshold; } // override PageNum if required if (indexTypeMapping.QueryOverrideSettings.DisableFullPageQuery && pagedQuery.PageNum == 0 && pagedQuery.PageSize != 0) { LoggingUtil.Log.InfoFormat( "Configuration rules require overriding PageNum to 1. AddressHistory {0}. Original Query Info: {1}", IndexServerUtils.FormatAddressHistory(messageContext.AddressHistory), FormatQueryInfo(pagedQuery)); pagedQuery.PageNum = 1; } } }
/// <summary> /// Validates the query. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="query">The query.</param> /// <param name="messageContext">The message context.</param> protected override void ValidateQuery(IndexTypeMapping indexTypeMapping, BaseMultiIndexIdQuery <SpanQueryResult> query, MessageContext messageContext) { SpanQuery spanQuery = query as SpanQuery; if (spanQuery == null) { throw new Exception(string.Format("Invalid span query, failed to cast")); } if (string.IsNullOrEmpty(spanQuery.TargetIndexName)) { spanQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(query.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + query.TargetIndexName); } if (query.IndexIdList == null || query.IndexIdList.Count == 0) { throw new Exception("No IndexIdList present on the query"); } if (query.PrimaryIdList != null && query.PrimaryIdList.Count != query.IndexIdList.Count) { throw new Exception("PrimaryIdList.Count does not match with IndexIdList.Count"); } if (spanQuery.Offset < 1 && spanQuery.Span != 0) { throw new Exception("SpanQuery.Offset should be greater than zero except when SpanQuery.Span is zero"); } }
/// <summary> /// Validates the query. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="getRangeQuery">The get range query.</param> private static void ValidateQuery(IndexTypeMapping indexTypeMapping, GetRangeQuery getRangeQuery) { if (string.IsNullOrEmpty(getRangeQuery.TargetIndexName)) { getRangeQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(getRangeQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + getRangeQuery.TargetIndexName); } if (getRangeQuery.IndexId == null || getRangeQuery.IndexId.Length == 0) { throw new Exception("No IndexId present on the GetRangeQuery"); } if ((getRangeQuery.Offset < 1) || (getRangeQuery.ItemNum < 1)) { throw new Exception("Both GetRangeQuery.Offset and GetRangeQuery.ItemNum should be greater than 1"); } }
/// <summary> /// Validates the query. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="firstLastQuery">The first last query.</param> private static void ValidateQuery(IndexTypeMapping indexTypeMapping, FirstLastQuery firstLastQuery) { if (string.IsNullOrEmpty(firstLastQuery.TargetIndexName)) { firstLastQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(firstLastQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + firstLastQuery.TargetIndexName); } if (firstLastQuery.IndexId == null || firstLastQuery.IndexId.Length == 0) { throw new Exception("No IndexId present on the FirstLastQuery"); } if ((firstLastQuery.FirstPageSize < 1) && (firstLastQuery.LastPageSize < 1)) { throw new Exception("Atleast one of FirstLastQuery.FirstPageSize and FirstLastQuery.LastPageSize should be greater than 1"); } }
/// <summary> /// Validates the query. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="randomQuery">The random query.</param> private static void ValidateQuery(IndexTypeMapping indexTypeMapping, RandomQuery randomQuery) { if (string.IsNullOrEmpty(randomQuery.TargetIndexName)) { randomQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(randomQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + randomQuery.TargetIndexName); } if (randomQuery.IndexId == null || randomQuery.IndexId.Length == 0) { throw new Exception("No IndexId present on the RandomQuery"); } if (randomQuery.Count < 1) { throw new Exception("Both RandomQuery.Count should be greater than 1"); } }
/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>Raw CacheIndexInternal</returns> internal static byte[] Process(MessageContext messageContext, IndexStoreContext storeContext) { short typeId; short messageTypeId = messageContext.TypeId; if (storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection.Contains(messageTypeId)) { typeId = messageTypeId; } else if (!storeContext.RelatedTypeIds.TryGetValue(messageTypeId, out typeId)) { LoggingUtil.Log.InfoFormat("Invalid TypeID for GetMessage {0}", messageTypeId); return(null); } return(storeContext.IndexStorageComponent.GetBuffer( typeId, new StorageKey( IndexServerUtils.FormExtendedId(messageContext.ExtendedId, 0), IndexCacheUtils.GeneratePrimaryId(messageContext.ExtendedId)))); }
/// <summary> /// Validates the query. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="query">The query.</param> /// <param name="messageContext">The message context.</param> protected override void ValidateQuery(IndexTypeMapping indexTypeMapping, BaseMultiIndexIdQuery <PagedIndexQueryResult> query, MessageContext messageContext) { if (string.IsNullOrEmpty(query.TargetIndexName)) { query.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(query.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + query.TargetIndexName); } if (query.IndexIdList == null || query.IndexIdList.Count == 0) { throw new Exception("No IndexIdList present on the query"); } if (query.PrimaryIdList != null && query.PrimaryIdList.Count != query.IndexIdList.Count) { throw new Exception("PrimaryIdList.Count does not match with IndexIdList.Count"); } PerformQueryOverride(indexTypeMapping, query, messageContext); }
/// <summary> /// Processes the specified query. /// </summary> /// <param name="query">The query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>Query Result</returns> internal TQueryResult Process(BaseMultiIndexIdQuery <TQueryResult> query, MessageContext messageContext, IndexStoreContext storeContext) { TQueryResult result; List <ResultItem> resultItemList = new List <ResultItem>(); Dictionary <byte[], IndexHeader> indexIdIndexHeaderMapping = null; bool isTagPrimarySort = false; string sortFieldName = null; List <SortOrder> sortOrderList = null; int totalCount = 0; int additionalAvailableItemCount = 0; GroupByResult groupByResult = null; StringBuilder exceptionInfo = new StringBuilder(); int indexCap = 0; IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; try { if (query.IndexIdList.Count > 0) { #region Validate Query ValidateQuery(indexTypeMapping, query, messageContext); #endregion #region Set sort vars Index targetIndexInfo = indexTypeMapping.IndexCollection[query.TargetIndexName]; indexCap = targetIndexInfo.MaxIndexSize; if (query.TagSort != null) { isTagPrimarySort = query.TagSort.IsTag; sortFieldName = query.TagSort.TagName; sortOrderList = new List <SortOrder>(1) { query.TagSort.SortOrder }; } else { isTagPrimarySort = targetIndexInfo.PrimarySortInfo.IsTag; sortFieldName = targetIndexInfo.PrimarySortInfo.FieldName; sortOrderList = targetIndexInfo.PrimarySortInfo.SortOrderList; } BaseComparer baseComparer = new BaseComparer(isTagPrimarySort, sortFieldName, sortOrderList); groupByResult = new GroupByResult(baseComparer); #endregion #region Prepare ResultList CacheIndexInternal targetIndex; IndexIdParams indexIdParam; int maxExtractCount; byte[] metadata; MetadataPropertyCollection metadataPropertyCollection; Dictionary <KeyValuePair <byte[], string>, CacheIndexInternal> internalIndexDictionary = new Dictionary <KeyValuePair <byte[], string>, CacheIndexInternal>(); int maxMergeCount = query.MaxMergeCount; IndexCondition queryIndexCondition = query.IndexCondition; for (int i = 0; i < query.IndexIdList.Count; i++) { #region Extract index and apply criteria indexIdParam = query.GetParamsForIndexId(query.IndexIdList[i]); maxExtractCount = ComputeMaxExtractCount(indexIdParam.MaxItems, query.GetAdditionalAvailableItemCount, indexIdParam.Filter, query.MaxMergeCount); // Note: This should be changed later and just extracted once if it is also requested in GetIndexHeader metadata = null; metadataPropertyCollection = null; if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, query.IndexIdList[i], storeContext, out metadata, out metadataPropertyCollection); } targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, (query.PrimaryIdList != null && i < query.PrimaryIdList.Count) ? query.PrimaryIdList[i] : IndexCacheUtils.GeneratePrimaryId(query.IndexIdList[i]), query.IndexIdList[i], targetIndexInfo.ExtendedIdSuffix, query.TargetIndexName, maxExtractCount, indexIdParam.Filter, true, queryIndexCondition, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, query.CapCondition, targetIndexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, query.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, query.GroupBy, false); #endregion if (targetIndex != null) { totalCount += targetIndex.OutDeserializationContext.TotalCount; additionalAvailableItemCount += targetIndex.Count; internalIndexDictionary.Add(new KeyValuePair <byte[], string>(query.IndexIdList[i], query.TargetIndexName), targetIndex); SetItemCounter(messageContext.TypeId, targetIndex.OutDeserializationContext); #region Dynamic tag sort if (query.TagSort != null) { targetIndex.Sort(query.TagSort); } #endregion #region Get items from index and merge if (query.GroupBy == null) { MergeAlgo.MergeItemLists(ref resultItemList, CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, int.MaxValue), query.MaxMergeCount, baseComparer); } else { MergeAlgo.MergeGroupResult(ref groupByResult, targetIndex.GroupByResult, query.MaxMergeCount, baseComparer); } if ((i != query.IndexIdList.Count - 1) && (resultItemList.Count == maxMergeCount)) { AdjustIndexCondition(GetConditionBoundaryBytes(resultItemList, groupByResult, query.GroupBy, isTagPrimarySort, sortFieldName), ref queryIndexCondition, baseComparer); } #endregion } } #endregion #region Subset Processing ProcessSubsets(query, ref resultItemList, ref groupByResult, baseComparer); #endregion #region Get Extra Tags for IndexIds in the list //Note: Getting extra tags from GroupByResult not supported for now if (query.TagsFromIndexes != null && query.TagsFromIndexes.Count != 0) { KeyValuePair <byte[] /*IndexId */, string /*IndexName*/> kvp; CacheIndexInternal additionalCacheIndexInternal; #region Form IndexId - PrimaryId Mapping Dictionary <byte[] /*IndexId */, int /*PrimaryId*/> indexIdPrimaryIdMapping = new Dictionary <byte[] /*IndexId */, int /*PrimaryId*/>(query.IndexIdList.Count, new ByteArrayEqualityComparer()); if (query.PrimaryIdList != null && query.PrimaryIdList.Count > 0) { //Form dictionary of IndexIdPrimaryIdMapping for (int i = 0; i < query.IndexIdList.Count && i < query.PrimaryIdList.Count; i++) { indexIdPrimaryIdMapping.Add(query.IndexIdList[i], query.PrimaryIdList[i]); } } #endregion int indexPrimaryId; foreach (ResultItem resultItem in resultItemList) { foreach (string indexName in query.TagsFromIndexes) { Index indexInfo = indexTypeMapping.IndexCollection[indexName]; kvp = new KeyValuePair <byte[], string>(resultItem.IndexId, indexName); if (!internalIndexDictionary.ContainsKey(kvp)) { additionalCacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, indexIdPrimaryIdMapping.TryGetValue(resultItem.IndexId, out indexPrimaryId) ? indexPrimaryId : IndexCacheUtils.GeneratePrimaryId(resultItem.IndexId), resultItem.IndexId, indexInfo.ExtendedIdSuffix, indexName, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null, indexInfo.IsMetadataPropertyCollection, null, query.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); if (additionalCacheIndexInternal != null) { SetItemCounter(messageContext.TypeId, additionalCacheIndexInternal.OutDeserializationContext); internalIndexDictionary.Add(kvp, additionalCacheIndexInternal); try { IndexServerUtils.GetTags(additionalCacheIndexInternal, resultItem, resultItem); } catch (Exception ex) { LoggingUtil.Log.Error(ex.ToString()); exceptionInfo.Append(" | " + ex.Message); } } } } } } #endregion #region Get IndexHeader if (query.GetIndexHeaderType == GetIndexHeaderType.AllIndexIds) { //Get IndexHeader for all IndexIds indexIdIndexHeaderMapping = new Dictionary <byte[], IndexHeader>(new ByteArrayEqualityComparer()); for (int i = 0; i < query.IndexIdList.Count; i++) { byte[] indexId = query.IndexIdList[i]; CacheIndexInternal targetIndexCacheIndexInternal; if (!indexIdIndexHeaderMapping.ContainsKey(indexId) && internalIndexDictionary.TryGetValue(new KeyValuePair <byte[], string>(indexId, query.TargetIndexName), out targetIndexCacheIndexInternal)) { indexIdIndexHeaderMapping.Add(indexId, GetIndexHeader(internalIndexDictionary, targetIndexCacheIndexInternal, indexId, query, indexTypeMapping, messageContext.TypeId, storeContext, i)); } } } else if (query.GetIndexHeaderType == GetIndexHeaderType.ResultItemsIndexIds) { //Get IndexHeader just for IndexIds present in the result indexIdIndexHeaderMapping = new Dictionary <byte[], IndexHeader>(new ByteArrayEqualityComparer()); if (query.GroupBy == null) { for (int i = 0; i < resultItemList.Count; i++) { ResultItem resultItem = resultItemList[i]; if (!indexIdIndexHeaderMapping.ContainsKey(resultItem.IndexId)) { CacheIndexInternal targetIndexCacheIndexInternal; internalIndexDictionary.TryGetValue(new KeyValuePair <byte[], string>(resultItem.IndexId, query.TargetIndexName), out targetIndexCacheIndexInternal); indexIdIndexHeaderMapping.Add(resultItem.IndexId, GetIndexHeader(internalIndexDictionary, targetIndexCacheIndexInternal, resultItem.IndexId, query, indexTypeMapping, messageContext.TypeId, storeContext, i)); } } } else { foreach (ResultItemBag resultItemBag in groupByResult) { for (int i = 0; i < resultItemBag.Count; i++) { ResultItem resultItem = resultItemBag[i]; if (!indexIdIndexHeaderMapping.ContainsKey(resultItem.IndexId)) { CacheIndexInternal targetIndexCacheIndexInternal; internalIndexDictionary.TryGetValue(new KeyValuePair <byte[], string>(resultItem.IndexId, query.TargetIndexName), out targetIndexCacheIndexInternal); indexIdIndexHeaderMapping.Add(resultItem.IndexId, GetIndexHeader(internalIndexDictionary, targetIndexCacheIndexInternal, resultItem.IndexId, query, indexTypeMapping, messageContext.TypeId, storeContext, i)); } } } } } #endregion #region Get data if (!query.ExcludeData) { DataTierUtil.GetData(resultItemList, groupByResult, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, query.FullDataIdInfo); } #endregion } result = new TQueryResult { ResultItemList = resultItemList, IndexIdIndexHeaderMapping = indexIdIndexHeaderMapping, TotalCount = totalCount, AdditionalAvailableItemCount = additionalAvailableItemCount, IsTagPrimarySort = isTagPrimarySort, SortFieldName = sortFieldName, SortOrderList = sortOrderList, IndexCap = indexCap, GroupByResult = groupByResult, ExceptionInfo = exceptionInfo.ToString() }; #region Log Potentially Bad Queries if (indexTypeMapping.QueryOverrideSettings != null && indexTypeMapping.QueryOverrideSettings.MaxResultItemsThresholdLog > 0 && resultItemList != null && resultItemList.Count > indexTypeMapping.QueryOverrideSettings.MaxResultItemsThresholdLog) { LoggingUtil.Log.ErrorFormat("Encountered potentially Bad Paged Query with Large Result Set of {0}. AddressHistory: {1}. Query Info: {2}", resultItemList.Count, IndexServerUtils.FormatAddressHistory(messageContext.AddressHistory), FormatQueryInfo(query)); } LoggingUtil.Log.DebugFormat("QueryInfo: {0}, AddressHistory: {1}", FormatQueryInfo(query), IndexServerUtils.FormatAddressHistory(messageContext.AddressHistory)); #endregion SetIndexIdListCounter(messageContext.TypeId, query); } catch (Exception ex) { exceptionInfo.Append(" | " + ex.Message); result = new TQueryResult { ExceptionInfo = exceptionInfo.ToString() }; LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing PagedIndexQuery : {1}", messageContext.TypeId, ex); } return(result); }
/// <summary> /// Gets the index header. /// </summary> /// <param name="internalIndexDictionary">The internal index dictionary.</param> /// <param name="targetIndexCacheIndexInternal">The targetindex CacheIndexInternal.</param> /// <param name="indexId">The index id.</param> /// <param name="query">The query.</param> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="typeId">The type id.</param> /// <param name="storeContext">The store context.</param> /// <param name="indexInIndexIdList">The indexInIndexIdList.</param> /// <returns>IndexHeader</returns> private static IndexHeader GetIndexHeader(Dictionary <KeyValuePair <byte[], string>, CacheIndexInternal> internalIndexDictionary, CacheIndexInternal targetIndexCacheIndexInternal, byte[] indexId, BaseMultiIndexIdQuery <TQueryResult> query, IndexTypeMapping indexTypeMapping, short typeId, IndexStoreContext storeContext, int indexInIndexIdList) { byte[] metadata = null; MetadataPropertyCollection metadataPropertyCollection = null; if (CheckMetaData(internalIndexDictionary, indexTypeMapping)) { if (indexTypeMapping.MetadataStoredSeperately) { #region Check if MetadataPropertyCollection is stored seperately IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, typeId, (query.PrimaryIdList != null && indexInIndexIdList < query.PrimaryIdList.Count) ? query.PrimaryIdList[indexInIndexIdList]: IndexCacheUtils.GeneratePrimaryId(indexId), indexId, storeContext, out metadata, out metadataPropertyCollection); #endregion } else { #region Check metadata on targetIndex if (indexTypeMapping.IndexCollection[query.TargetIndexName].MetadataPresent) { if (indexTypeMapping.IndexCollection[query.TargetIndexName].IsMetadataPropertyCollection) { metadataPropertyCollection = targetIndexCacheIndexInternal.MetadataPropertyCollection; } else { metadata = targetIndexCacheIndexInternal.Metadata; } } #endregion #region Check metadata on other extracted indexes if (query.TagsFromIndexes != null) { foreach (string indexName in query.TagsFromIndexes) { if (indexTypeMapping.IndexCollection[indexName].MetadataPresent) { if (indexTypeMapping.IndexCollection[indexName].IsMetadataPropertyCollection) { metadataPropertyCollection = internalIndexDictionary[new KeyValuePair <byte[], string>(indexId, indexName)]. MetadataPropertyCollection; } else { metadata = internalIndexDictionary[new KeyValuePair <byte[], string>(indexId, indexName)]. Metadata; } } } } #endregion } } return(new IndexHeader { Metadata = metadata, MetadataPropertyCollection = metadataPropertyCollection, VirtualCount = targetIndexCacheIndexInternal.VirtualCount }); }
/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MessageContext messageContext, IndexStoreContext storeContext) { lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId)) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; List <RelayMessage> indexStorageMessageList = new List <RelayMessage>(indexTypeMapping.IndexCollection.Count); List <RelayMessage> dataStorageMessageList = new List <RelayMessage>(); CacheIndexInternal internalIndex; List <byte[]> fullDataIdList; if (indexTypeMapping.MetadataStoredSeperately) { BinaryStorageAdapter.Delete( storeContext.IndexStorageComponent, messageContext.TypeId, messageContext.PrimaryId, messageContext.ExtendedId); } foreach (Index index in indexTypeMapping.IndexCollection) { internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, messageContext.ExtendedId, index.ExtendedIdSuffix, index.IndexName, 0, null, true, null, false, false, index.PrimarySortInfo, index.LocalIdentityTagList, index.StringHashCodeDictionary, null, indexTypeMapping.IndexCollection[index.IndexName].IsMetadataPropertyCollection, null, DomainSpecificProcessingType.None, null, null, null, true); if (internalIndex != null) { #region Deletes messages for data store if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode)) { fullDataIdList = DataTierUtil.GetFullDataIds(messageContext.ExtendedId, internalIndex.InternalItemList, indexTypeMapping.FullDataIdFieldList); short relatedTypeId; foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { if (storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } else { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } } } } #endregion #region Delete messages for index store BinaryStorageAdapter.Delete( storeContext.IndexStorageComponent, messageContext.TypeId, messageContext.PrimaryId, IndexServerUtils.FormExtendedId(messageContext.ExtendedId, index.ExtendedIdSuffix)); #endregion } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } } }
/// <summary> /// Processes the specified filtered index delete command. /// </summary> /// <param name="filteredIndexDeleteCommand">The filtered index delete command.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process( FilteredIndexDeleteCommand filteredIndexDeleteCommand, MessageContext messageContext, IndexStoreContext storeContext) { if (filteredIndexDeleteCommand != null) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; Index indexInfo = indexTypeMapping.IndexCollection[filteredIndexDeleteCommand.TargetIndexName]; byte[] metadata = null; MetadataPropertyCollection metadataPropertyCollection = null; #region Get CacheIndexInternal for TargetIndexName if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, filteredIndexDeleteCommand.IndexId, storeContext, out metadata, out metadataPropertyCollection); } CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, filteredIndexDeleteCommand.IndexId, indexInfo.ExtendedIdSuffix, filteredIndexDeleteCommand.TargetIndexName, 0, filteredIndexDeleteCommand.DeleteFilter, false, null, false, true, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null, indexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, DomainSpecificProcessingType.None, null, null, null, false); #endregion if (cacheIndexInternal != null) { #region Increment perf counters' number PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsFilteredPerFilterDeleteRequest, messageContext.TypeId, (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count)); #endregion #region Update VirtualCount cacheIndexInternal.VirtualCount -= (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.Count); #endregion #region Save CacheIndexInternal to local storage since item which pass delete filter are pruned in it byte[] extendedId = IndexServerUtils.FormExtendedId(filteredIndexDeleteCommand.IndexId, indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix); // compose a bdb entry header bool isCompress = storeContext.GetCompressOption(messageContext.TypeId); PayloadStorage bdbEntryHeader = new PayloadStorage { Compressed = isCompress, TTL = -1, LastUpdatedTicks = DateTime.Now.Ticks, ExpirationTicks = -1, Deactivated = false }; BinaryStorageAdapter.Save( storeContext.MemoryPool, storeContext.IndexStorageComponent, messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, extendedId, bdbEntryHeader, Serializer.Serialize <CacheIndexInternal>(cacheIndexInternal, isCompress, RelayMessage.RelayCompressionImplementation)); #endregion #region Data store deletes if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode) && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList != null && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count > 0) { List <RelayMessage> dataStorageMessageList = new List <RelayMessage>(); short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } cacheIndexInternal.InternalItemList = cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList; List <byte[]> fullDataIdList = DataTierUtil.GetFullDataIds(cacheIndexInternal.InDeserializationContext.IndexId, cacheIndexInternal.InternalItemList, indexTypeMapping.FullDataIdFieldList); foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } } #endregion } } }
/// <summary> /// Processes the specified random query. /// </summary> /// <param name="randomQuery">The random query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>RandomQueryResult</returns> internal static RandomQueryResult Process(RandomQuery randomQuery, MessageContext messageContext, IndexStoreContext storeContext) { RandomQueryResult randomQueryResult; List <ResultItem> resultItemList = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; byte[] metadata = null; MetadataPropertyCollection metadataPropertyCollection = null; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Validate Query ValidateQuery(indexTypeMapping, randomQuery); #endregion Index targetIndexInfo = indexTypeMapping.IndexCollection[randomQuery.TargetIndexName]; int indexCap = targetIndexInfo.MaxIndexSize; #region Prepare Result #region Extract index and apply criteria if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, randomQuery.IndexId, storeContext, out metadata, out metadataPropertyCollection); } CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, randomQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, randomQuery.TargetIndexName, 0, randomQuery.Filter, true, randomQuery.IndexCondition, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null, targetIndexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, randomQuery.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); #endregion if (targetIndex != null) { indexSize = targetIndex.OutDeserializationContext.TotalCount; virtualCount = targetIndex.VirtualCount; indexExists = true; // update performance counters PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerRandomQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerRandomQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); #region Populate resultLists int indexItemCount = targetIndex.Count; int resultListCount = Math.Min(randomQuery.Count, indexItemCount); IEnumerable <int> itemPositionList = Algorithm.RandomSubset(new Random(), 0, indexItemCount - 1, resultListCount); resultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, itemPositionList); #endregion #region Get data if (!randomQuery.ExcludeData) { DataTierUtil.GetData(resultItemList, null, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, randomQuery.FullDataIdInfo); } #endregion #region Get metadata if (randomQuery.GetMetadata && !indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredWithIndex(indexTypeMapping, new List <CacheIndexInternal>(1) { targetIndex }, out metadata, out metadataPropertyCollection); } #endregion #endregion } randomQueryResult = new RandomQueryResult(indexExists, indexSize, metadata, metadataPropertyCollection, resultItemList, virtualCount, indexCap, null); } catch (Exception ex) { randomQueryResult = new RandomQueryResult(false, -1, null, null, null, -1, 0, ex.Message); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing RandomQuery : {1}", messageContext.TypeId, ex); } return(randomQueryResult); }
/// <summary> /// Processes the specified first last query. /// </summary> /// <param name="firstLastQuery">The first last query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>FirstLastQueryResult</returns> internal static FirstLastQueryResult Process(FirstLastQuery firstLastQuery, MessageContext messageContext, IndexStoreContext storeContext) { FirstLastQueryResult firstLastQueryResult; List <ResultItem> firstPageResultItemList = null; List <ResultItem> lastPageResultItemList = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; byte[] metadata = null; MetadataPropertyCollection metadataPropertyCollection = null; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Validate Query ValidateQuery(indexTypeMapping, firstLastQuery); #endregion int maxItemsPerIndex = ((firstLastQuery.FirstPageSize > 0) && (firstLastQuery.LastPageSize < 1)) ? firstLastQuery.FirstPageSize : int.MaxValue; Index targetIndexInfo = indexTypeMapping.IndexCollection[firstLastQuery.TargetIndexName]; int indexCap = targetIndexInfo.MaxIndexSize; #region Prepare Result #region Extract index and apply criteria if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, firstLastQuery.IndexId, storeContext, out metadata, out metadataPropertyCollection); } CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, firstLastQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, firstLastQuery.TargetIndexName, maxItemsPerIndex, firstLastQuery.Filter, true, firstLastQuery.IndexCondition, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null, targetIndexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, firstLastQuery.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); #endregion if (targetIndex != null) { indexSize = targetIndex.OutDeserializationContext.TotalCount; virtualCount = targetIndex.VirtualCount; indexExists = true; #region Dynamic tag sort if (firstLastQuery.TagSort != null) { targetIndex.Sort(firstLastQuery.TagSort); } #endregion // update perf counters PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerFirstLastQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerFirstLastQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); #region Populate resultLists if (firstLastQuery.FirstPageSize + firstLastQuery.LastPageSize <= targetIndex.Count) { firstPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, firstLastQuery.FirstPageSize); lastPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, targetIndex.Count - firstLastQuery.LastPageSize + 1, firstLastQuery.LastPageSize); } else { //Populate everything in firstPageResultItemList firstPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, targetIndex.Count); } #endregion #region Get data if (!firstLastQuery.ExcludeData) { //First Page if (firstPageResultItemList != null) { DataTierUtil.GetData(firstPageResultItemList, null, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, firstLastQuery.FullDataIdInfo); } //Last Page if (lastPageResultItemList != null) { DataTierUtil.GetData(lastPageResultItemList, null, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, firstLastQuery.FullDataIdInfo); } } #endregion #region Get metadata if (firstLastQuery.GetMetadata && !indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredWithIndex(indexTypeMapping, new List <CacheIndexInternal>(1) { targetIndex }, out metadata, out metadataPropertyCollection); } #endregion } #endregion firstLastQueryResult = new FirstLastQueryResult(indexExists, indexSize, metadata, metadataPropertyCollection, firstPageResultItemList, lastPageResultItemList, virtualCount, indexCap, null); } catch (Exception ex) { firstLastQueryResult = new FirstLastQueryResult(false, -1, null, null, null, null, -1, 0, ex.Message); LoggingUtil.Log.ErrorFormat("TypeID {0} -- Error processing FirstLastQuery : {1}", messageContext.TypeId, ex); } return(firstLastQueryResult); }
/// <summary> /// Processes the specified intersection query. /// </summary> /// <param name="intersectionQuery">The intersection query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>IntersectionQueryResult</returns> internal static IntersectionQueryResult Process(IntersectionQuery intersectionQuery, MessageContext messageContext, IndexStoreContext storeContext) { //Fetch each index (assume all indexes are local) and perform intersection and return the results IntersectionQueryResult intersectionQueryResult; List <IndexDataItem> resultItemList = null; Dictionary <byte[], IndexHeader> indexIdIndexHeaderMapping = new Dictionary <byte[], IndexHeader>(new ByteArrayEqualityComparer()); List <string> localIdentityTagNames = null; bool isTagPrimarySort = false; string sortFieldName = null; List <SortOrder> sortOrderList = null; StringBuilder exceptionInfo = new StringBuilder(); try { if (intersectionQuery.IndexIdList.Count > 0) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; ValidateQuery(indexTypeMapping, intersectionQuery); #region Set sort vars Index targetIndexInfo = indexTypeMapping.IndexCollection[intersectionQuery.TargetIndexName]; localIdentityTagNames = indexTypeMapping.IndexCollection[intersectionQuery.TargetIndexName].LocalIdentityTagList; bool sortFieldPartOfLocalId = IsSortFieldPartOfLocalId(localIdentityTagNames, targetIndexInfo.PrimarySortInfo); TagSort itemIdTagSort = new TagSort("ItemId", false, new SortOrder(DataType.Int32, SortBy.ASC)); if (!sortFieldPartOfLocalId) { //Set sort vars isTagPrimarySort = itemIdTagSort.IsTag; sortFieldName = itemIdTagSort.TagName; sortOrderList = new List <SortOrder>(1) { itemIdTagSort.SortOrder }; } else { isTagPrimarySort = targetIndexInfo.PrimarySortInfo.IsTag; sortFieldName = targetIndexInfo.PrimarySortInfo.FieldName; sortOrderList = targetIndexInfo.PrimarySortInfo.SortOrderList; } #endregion #region Fetch CacheIndexInternal and Intersect CacheIndexInternal targetIndex; CacheIndexInternal resultCacheIndexInternal = null; IntersectionQueryParams indexIdParam; byte[] indexId; byte[] metadata; MetadataPropertyCollection metadataPropertyCollection; for (int i = 0; i < intersectionQuery.IndexIdList.Count; i++) { #region Extract index and apply criteria indexId = intersectionQuery.IndexIdList[i]; indexIdParam = intersectionQuery.GetIntersectionQueryParamForIndexId(indexId); // Note: This should be changed later and just extracted once if it is also requested in GetIndexHeader metadata = null; metadataPropertyCollection = null; if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, indexId, storeContext, out metadata, out metadataPropertyCollection); } targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, (intersectionQuery.PrimaryIdList != null && i < intersectionQuery.PrimaryIdList.Count) ? intersectionQuery.PrimaryIdList[i] : IndexCacheUtils.GeneratePrimaryId(indexId), indexId, targetIndexInfo.ExtendedIdSuffix, intersectionQuery.TargetIndexName, indexIdParam.Count, indexIdParam.Filter, true, indexIdParam.IndexCondition, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null, targetIndexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, intersectionQuery.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); #endregion if (targetIndex != null) { if (targetIndex.Count <= 0) { // No items in one of the indexes. Stop Interestion !! resultCacheIndexInternal = null; break; } PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerIntersectionQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerIntersectionQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); if (!sortFieldPartOfLocalId) { //Need to sort indexes by item id targetIndex.Sort(itemIdTagSort); } #region Intersection if (resultCacheIndexInternal == null) { // No need to perform intersection for first index resultCacheIndexInternal = targetIndex; } else { IntersectionAlgo.Intersect(isTagPrimarySort, sortFieldName, localIdentityTagNames, sortOrderList, resultCacheIndexInternal.InternalItemList, targetIndex.InternalItemList, intersectionQuery.MaxResultItems, i == intersectionQuery.IndexIdList.Count - 1 ? intersectionQuery.IsSingleClusterQuery : false); if (resultCacheIndexInternal == null || resultCacheIndexInternal.Count < 1) { // Unable to fetch one of the indexes. Stop Interestion !! resultCacheIndexInternal = null; indexIdIndexHeaderMapping = null; break; } } #endregion } else { // Unable to fetch one of the indexes. Stop Interestion !! resultCacheIndexInternal = null; indexIdIndexHeaderMapping = null; break; } #region Get MetaData if (intersectionQuery.GetIndexHeader) { if (!indexIdIndexHeaderMapping.ContainsKey(indexId)) { indexIdIndexHeaderMapping.Add(indexId, IndexServerUtils.GetIndexHeader(targetIndex, indexTypeMapping, messageContext.TypeId, IndexCacheUtils.GeneratePrimaryId(indexId), storeContext)); } } #endregion } if (resultCacheIndexInternal != null && resultCacheIndexInternal.Count > 0) { resultItemList = CacheIndexInternalAdapter.GetIndexDataItemList(resultCacheIndexInternal, 1, int.MaxValue); } #endregion #region Get data if (!intersectionQuery.ExcludeData && resultItemList != null && resultItemList.Count > 0) { DataTierUtil.GetData(resultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, intersectionQuery.FullDataIdInfo); } #endregion } intersectionQueryResult = new IntersectionQueryResult(resultItemList, indexIdIndexHeaderMapping, localIdentityTagNames, isTagPrimarySort, sortFieldName, sortOrderList, exceptionInfo.ToString()); // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.IndexLookupAvgPerIntersectionQuery, messageContext.TypeId, intersectionQuery.IndexIdList.Count); } catch (Exception ex) { exceptionInfo.Append(" | " + ex.Message); intersectionQueryResult = new IntersectionQueryResult(null, null, null, false, null, null, exceptionInfo.ToString()); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing IntersectionQuery : {1}", messageContext.TypeId, ex); } return(intersectionQueryResult); }
internal static MultiIndexContainsQueryResult Process(MultiIndexContainsQuery multiIndexContainsQuery, MessageContext messageContext, IndexStoreContext storeContext) { MultiIndexContainsQueryResult multiIndexContainsQueryResult; List <Pair <MultiIndexContainsQueryResultItem, IndexHeader> > multiIndexContainsQueryResultItemIndexHeaderList = new List <Pair <MultiIndexContainsQueryResultItem, IndexHeader> >(multiIndexContainsQuery.IndexIdList.Count); int indexSize = -1; int indexCap = 0; StringBuilder exceptionInfo = new StringBuilder(); try { if (multiIndexContainsQuery.IndexIdList.Count > 0) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; short relatedTypeId; ValidateQuery(multiIndexContainsQuery, indexTypeMapping, storeContext, messageContext.TypeId, out relatedTypeId); Index targetIndexInfo = indexTypeMapping.IndexCollection[multiIndexContainsQuery.TargetIndexName]; indexCap = targetIndexInfo.MaxIndexSize; MultiIndexContainsQueryParams multiIndexContainsQueryParam; int searchIndex; IndexDataItem indexDataItem; MultiIndexContainsQueryResultItem multiIndexContainsQueryResultItem; IndexHeader indexHeader; List <RelayMessage> dataStoreMessages = new List <RelayMessage>(); byte[] extendedId; byte[] metadata; MetadataPropertyCollection metadataPropertyCollection; foreach (byte[] indexId in multiIndexContainsQuery.IndexIdList) { #region Get TargetIndex // Note: This should be changed later and just extracted once if it is also requested in GetIndexHeader metadata = null; metadataPropertyCollection = null; if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, IndexCacheUtils.GeneratePrimaryId(indexId), indexId, storeContext, out metadata, out metadataPropertyCollection); } multiIndexContainsQueryParam = multiIndexContainsQuery.GetMultiIndexContainsQueryParamForIndexId(indexId); CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, IndexCacheUtils.GeneratePrimaryId(indexId), indexId, targetIndexInfo.ExtendedIdSuffix, multiIndexContainsQuery.TargetIndexName, multiIndexContainsQueryParam.Count, multiIndexContainsQueryParam.Filter, true, multiIndexContainsQueryParam.IndexCondition, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null, targetIndexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, multiIndexContainsQuery.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); #endregion if (cacheIndexInternal != null) { indexSize = cacheIndexInternal.OutDeserializationContext.TotalCount; multiIndexContainsQueryResultItem = null; indexHeader = null; // update the performance counter PerformanceCounters.Instance.SetCounterValue(PerformanceCounterEnum.NumOfItemsInIndexPerMultiIndexContainsQuery, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue(PerformanceCounterEnum.NumOfItemsReadPerMultiIndexContainsQuery, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); #region MultiIndexContainsQueryResultItem foreach (IndexItem queryIndexItem in multiIndexContainsQuery.IndexItemList) { #region Search item in index searchIndex = cacheIndexInternal.Search(queryIndexItem); #endregion if (searchIndex > -1) { if (multiIndexContainsQueryResultItem == null) { multiIndexContainsQueryResultItem = new MultiIndexContainsQueryResultItem { IndexId = indexId, IndexCap = indexCap, IndexExists = true, IndexSize = indexSize }; } indexDataItem = new IndexDataItem( InternalItemAdapter.ConvertToIndexItem(cacheIndexInternal.GetItem(searchIndex), cacheIndexInternal.InDeserializationContext)); multiIndexContainsQueryResultItem.Add(indexDataItem); // Data if (!multiIndexContainsQuery.ExcludeData) { extendedId = DataTierUtil.GetFullDataId(indexId, indexDataItem, multiIndexContainsQuery.FullDataIdInfo != null && multiIndexContainsQuery.FullDataIdInfo.RelatedTypeName != null ? multiIndexContainsQuery.FullDataIdInfo.FullDataIdFieldList : indexTypeMapping.FullDataIdFieldList); dataStoreMessages.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(extendedId), extendedId, MessageType.Get)); } } } #endregion #region Add indexHeader to Result if (multiIndexContainsQueryResultItem != null && multiIndexContainsQueryResultItem.Count > 0) { if (multiIndexContainsQuery.GetIndexHeader) { indexHeader = IndexServerUtils.GetIndexHeader(cacheIndexInternal, indexTypeMapping, messageContext.TypeId, IndexCacheUtils.GeneratePrimaryId(indexId), storeContext); } multiIndexContainsQueryResultItemIndexHeaderList.Add(new Pair <MultiIndexContainsQueryResultItem, IndexHeader> { First = multiIndexContainsQueryResultItem, Second = indexHeader }); } #endregion } } #region Get data if (!multiIndexContainsQuery.ExcludeData) { storeContext.ForwarderComponent.HandleMessages(dataStoreMessages); int i = 0; foreach ( Pair <MultiIndexContainsQueryResultItem, IndexHeader> pair in multiIndexContainsQueryResultItemIndexHeaderList) { foreach (IndexDataItem resultItem in pair.First) { if (dataStoreMessages[i].Payload != null) { resultItem.Data = dataStoreMessages[i].Payload.ByteArray; } i++; } } } #endregion } multiIndexContainsQueryResult = new MultiIndexContainsQueryResult(multiIndexContainsQueryResultItemIndexHeaderList, exceptionInfo.ToString()); // update performance counter PerformanceCounters.Instance.SetCounterValue(PerformanceCounterEnum.IndexLookupAvgPerMultiIndexContainsQuery, messageContext.TypeId, multiIndexContainsQuery.IndexIdList.Count); } catch (Exception ex) { multiIndexContainsQueryResult = new MultiIndexContainsQueryResult(null, ex.Message); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing MultiIndexContainsQuery : {1}", messageContext.TypeId, ex); } return(multiIndexContainsQueryResult); }
/// <summary> /// Processes the specified MetadataPropertyQuery. /// </summary> /// <param name="metadataPropertyQuery">The MetadataPropertyQuery.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns></returns> internal static MetadataPropertyQueryResult Process(MetadataPropertyQuery metadataPropertyQuery, MessageContext messageContext, IndexStoreContext storeContext) { MetadataPropertyQueryResult metadataPropertyQueryResult; MetadataPropertyCollection metadataPropertyCollection = null; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; if (indexTypeMapping.IsMetadataPropertyCollection || indexTypeMapping.IndexCollection[metadataPropertyQuery.TargetIndexName].IsMetadataPropertyCollection) { #region Fetch MetadataPropertyCollection if (indexTypeMapping.MetadataStoredSeperately) { byte[] metadata; IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, metadataPropertyQuery.PrimaryId, metadataPropertyQuery.IndexId, storeContext, out metadata, out metadataPropertyCollection); } else { // Get CacheIndexInternal Index indexInfo = indexTypeMapping.IndexCollection[metadataPropertyQuery.TargetIndexName]; CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext. TypeId, metadataPropertyQuery .PrimaryId, metadataPropertyQuery .IndexId, indexInfo. ExtendedIdSuffix, metadataPropertyQuery . TargetIndexName, 0, null, false, null, true, false, null, null, null, null, true, null, DomainSpecificProcessingType .None, null, null, null, true); if (cacheIndexInternal != null) { metadataPropertyCollection = cacheIndexInternal.MetadataPropertyCollection; } } #endregion } metadataPropertyQueryResult = new MetadataPropertyQueryResult { MetadataPropertyCollection = metadataPropertyCollection }; } catch (Exception ex) { metadataPropertyQueryResult = new MetadataPropertyQueryResult { MetadataPropertyCollection = metadataPropertyCollection, ExceptionInfo = ex.Message }; LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing MetadataPropertyQuery : {1} for IndexId : {2} and TargetIndexname : {3}", messageContext.TypeId, ex, metadataPropertyQuery.IndexId != null ? IndexCacheUtils.GetReadableByteArray(metadataPropertyQuery.IndexId) : "Null", metadataPropertyQuery.TargetIndexName ?? "Null"); } return(metadataPropertyQueryResult); }
/// <summary> /// Processes the specified cache index. /// </summary> /// <param name="cacheIndex">Index of the cache.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(CacheIndex cacheIndex, MessageContext messageContext, IndexStoreContext storeContext) { lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId)) { try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Extract CacheIndex and Validate from incoming message ValidateSave(cacheIndex); #endregion #region Log CacheIndex before processing StringBuilder dbgIndexInfo = null; if (LoggingUtil.Log.IsDebugEnabled) { dbgIndexInfo = new StringBuilder(); dbgIndexInfo.Append("TypeId=").Append(messageContext.TypeId).Append(Environment.NewLine); dbgIndexInfo.Append(IndexServerUtils.GetPrintableCacheIndex(cacheIndex, storeContext.TagHashCollection, messageContext.TypeId)); } #endregion #region Init vars List <RelayMessage> indexStorageMessageList = new List <RelayMessage>(); List <RelayMessage> dataStorageMessageList = new List <RelayMessage>(); List <CacheIndexInternal> internalIndexList = new List <CacheIndexInternal>(); List <IndexItem> cappedDeleteItemList = new List <IndexItem>(); CacheIndexInternal internalIndex; #endregion if (cacheIndex.IndexVirtualCountMapping == null) { #region Save Items #region Extract CacheIndexInternal from index storage if (cacheIndex.TargetIndexName == null) //Save to multiple indexes { #region Get CacheIndexInternal for multiple indexes foreach (KeyValuePair <string /*IndexName*/, List <string> /*TagNameList*/> kvp in cacheIndex.IndexTagMapping) { Index indexInfo = indexTypeMapping.IndexCollection[kvp.Key]; internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, indexInfo.ExtendedIdSuffix, kvp.Key, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (internalIndex != null) { // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumberOfItemsInIndexPerSave, messageContext.TypeId, internalIndex.OutDeserializationContext.TotalCount); } if (internalIndex == null || cacheIndex.ReplaceFullIndex) //CacheIndexInternal does not exists or is to be discarded { internalIndex = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext { TypeId = messageContext.TypeId, TagHashCollection = storeContext.TagHashCollection, IndexId = cacheIndex.IndexId, IndexName = kvp.Key, InclusiveFilter = true, PrimarySortInfo = indexInfo.PrimarySortInfo, LocalIdentityTagNames = indexInfo.LocalIdentityTagList, StringHashCollection = storeContext.StringHashCollection, StringHashCodeDictionary = indexInfo.StringHashCodeDictionary } }; } internalIndexList.Add(internalIndex); } #endregion } else //Save to single index { #region Get CacheIndexInternal for TargetIndexName Index indexInfo = indexTypeMapping.IndexCollection[cacheIndex.TargetIndexName]; internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, indexInfo.ExtendedIdSuffix, cacheIndex.TargetIndexName, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (internalIndex != null) { // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumberOfItemsInIndexPerSave, messageContext.TypeId, internalIndex.OutDeserializationContext.TotalCount); } if (internalIndex == null || cacheIndex.ReplaceFullIndex) //CacheIndexInternal does not exists or is to be discarded { internalIndex = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext { TypeId = messageContext.TypeId, TagHashCollection = storeContext.TagHashCollection, IndexId = cacheIndex.IndexId, IndexName = cacheIndex.TargetIndexName, InclusiveFilter = true, PrimarySortInfo = indexInfo.PrimarySortInfo, LocalIdentityTagNames = indexInfo.LocalIdentityTagList, StringHashCollection = storeContext.StringHashCollection, StringHashCodeDictionary = indexInfo.StringHashCodeDictionary } }; } internalIndexList.Add(internalIndex); #endregion } #endregion #region Log CacheIndexInternals before save if (LoggingUtil.Log.IsDebugEnabled && dbgIndexInfo != null) { dbgIndexInfo.Append(Environment.NewLine).Append(string.Format("BEFORE SAVE {0}", IndexServerUtils.GetPrintableCacheIndexInternalList( internalIndexList, storeContext.TagHashCollection, messageContext.TypeId))); } #endregion #region Process Delete and Add List try { #region Process Delete List if (cacheIndex.DeleteList.Count > 0 && !cacheIndex.ReplaceFullIndex) { ProcessDeleteList(internalIndexList, cacheIndex.DeleteList, messageContext.TypeId); } #endregion #region Process Add List if (cacheIndex.AddList.Count > 0 || cacheIndex.UpdateMetadata) { ProcessAddList(internalIndexList, cappedDeleteItemList, cacheIndex, storeContext, indexTypeMapping); } #endregion } catch { LoggingUtil.Log.Debug(IndexServerUtils.GetPrintableCacheIndexInternalList(internalIndexList, storeContext.TagHashCollection, messageContext.TypeId)); throw; } #endregion #region Log CacheIndexInternals after save if (LoggingUtil.Log.IsDebugEnabled && dbgIndexInfo != null) { dbgIndexInfo.Append(Environment.NewLine).Append(string.Format("AFTER SAVE {0}", IndexServerUtils.GetPrintableCacheIndexInternalList(internalIndexList, storeContext.TagHashCollection, messageContext.TypeId))); } #endregion #region Data store relay messages for deletes and saves if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode) && !cacheIndex.PreserveData) { byte[] fullDataId; short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } #region Delete Messages foreach (IndexItem indexItem in cacheIndex.DeleteList) { fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexItem, indexTypeMapping.FullDataIdFieldList); if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } #endregion #region Save Messages foreach (IndexDataItem indexDataItem in cacheIndex.AddList) { fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexDataItem, indexTypeMapping.FullDataIdFieldList); if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, DateTime.Now, indexDataItem.Data ?? new byte[0], storeContext.GetCompressOption(messageContext.TypeId), MessageType.Save)); if (indexDataItem.Data == null || indexDataItem.Data.Length == 0) { LoggingUtil.Log.WarnFormat("Saving null data for TypeId: {0}, IndexId: {1}, ItemId: {2}, FullDataId: {3}, PrimaryId: {4}", relatedTypeId, IndexCacheUtils.GetReadableByteArray(cacheIndex.IndexId), IndexCacheUtils.GetReadableByteArray(indexDataItem.ItemId), IndexCacheUtils.GetReadableByteArray(fullDataId), IndexCacheUtils.GeneratePrimaryId(fullDataId)); } } } #endregion #region Capped Item Delete Messages foreach (IndexItem indexItem in cappedDeleteItemList) { fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexItem, indexTypeMapping.FullDataIdFieldList); if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } #endregion #region Send relay mesaages to data store if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } #endregion } #endregion #endregion if (dbgIndexInfo != null) { LoggingUtil.Log.Debug(dbgIndexInfo.ToString()); } } else { #region Update Virtual Count foreach (KeyValuePair <string /*IndexName*/, int /*VirtualCount*/> kvp in cacheIndex.IndexVirtualCountMapping) { Index indexInfo = indexTypeMapping.IndexCollection[kvp.Key]; internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, indexInfo.ExtendedIdSuffix, kvp.Key, 0, null, true, null, true, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (internalIndex == null) { internalIndex = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext { TypeId = messageContext.TypeId, TagHashCollection = storeContext.TagHashCollection, IndexId = cacheIndex.IndexId, IndexName = kvp.Key, InclusiveFilter = true, DeserializeHeaderOnly = true, PrimarySortInfo = indexInfo.PrimarySortInfo, LocalIdentityTagNames = indexInfo.LocalIdentityTagList, StringHashCollection = storeContext.StringHashCollection, StringHashCodeDictionary = indexInfo.StringHashCodeDictionary } }; } else { // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumberOfItemsInIndexPerSave, messageContext.TypeId, internalIndex.OutDeserializationContext.TotalCount); } internalIndex.VirtualCount = kvp.Value; internalIndexList.Add(internalIndex); } #endregion } #region Index storage relay messages for each CacheIndexInternal #region Metadata if (indexTypeMapping.MetadataStoredSeperately && cacheIndex.UpdateMetadata) { indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, DateTime.Now, cacheIndex.Metadata ?? new byte[0], storeContext.GetCompressOption(messageContext.TypeId), MessageType.Save)); } #endregion #region Index(es) byte[] payload; CompactBinaryWriter writer; RelayMessage indexStorageMessage; byte[] extendedId; foreach (CacheIndexInternal cacheIndexInternal in internalIndexList) { extendedId = IndexServerUtils.FormExtendedId( cacheIndex.IndexId, indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix); // This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal // ** TBD - Should be removed later if (LegacySerializationUtil.Instance.IsSupported(messageContext.TypeId)) { writer = new CompactBinaryWriter(new BinaryWriter(new MemoryStream())); cacheIndexInternal.Serialize(writer); payload = new byte[writer.BaseStream.Length]; writer.BaseStream.Position = 0; writer.BaseStream.Read(payload, 0, payload.Length); indexStorageMessage = new RelayMessage(messageContext.TypeId, cacheIndex.PrimaryId, extendedId, DateTime.Now, payload, storeContext.GetCompressOption(messageContext.TypeId), MessageType.Save); } else { indexStorageMessage = RelayMessage.GetSaveMessageForObject(messageContext.TypeId, cacheIndex.PrimaryId, extendedId, DateTime.Now, cacheIndexInternal, storeContext.GetCompressOption(messageContext.TypeId)); } indexStorageMessageList.Add(indexStorageMessage); } #endregion #region Send relay mesaages to index storage storeContext.IndexStorageComponent.HandleMessages(indexStorageMessageList); #endregion #endregion } catch (Exception ex) { LoggingUtil.Log.DebugFormat("CacheIndex: {0}", IndexServerUtils.GetPrintableCacheIndex(cacheIndex, storeContext.TagHashCollection, messageContext.TypeId)); throw new Exception("TypeId " + messageContext.TypeId + " -- Error processing save message.", ex); } } }
/// <summary> /// Processes the specified contains index query. /// </summary> /// <param name="containsIndexQuery">The contains index query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>ContainsIndexQueryResult</returns> internal static ContainsIndexQueryResult Process(ContainsIndexQuery containsIndexQuery, MessageContext messageContext, IndexStoreContext storeContext) { ContainsIndexQueryResult containsIndexQueryResult; MultiItemResult multiItemResult = null; byte[] metadata = null; MetadataPropertyCollection metadataPropertyCollection = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Check TargetIndexName if (string.IsNullOrEmpty(containsIndexQuery.TargetIndexName)) { containsIndexQuery.TargetIndexName = IndexServerUtils.CheckQueryTargetIndexName(indexTypeMapping); } if (!indexTypeMapping.IndexCollection.Contains(containsIndexQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + containsIndexQuery.TargetIndexName); } #endregion Index targetIndexInfo = indexTypeMapping.IndexCollection[containsIndexQuery.TargetIndexName]; int indexCap = targetIndexInfo.MaxIndexSize; List <CacheIndexInternal> internalCacheIndexList = new List <CacheIndexInternal>(); #region Get TargetIndex CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, containsIndexQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, containsIndexQuery.TargetIndexName, 0, null, true, null, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null, targetIndexInfo.IsMetadataPropertyCollection, null, containsIndexQuery.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); #endregion if (cacheIndexInternal != null) { internalCacheIndexList.Add(cacheIndexInternal); indexExists = true; indexSize = cacheIndexInternal.OutDeserializationContext.TotalCount; virtualCount = cacheIndexInternal.VirtualCount; // update the performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); int searchIndex; IndexDataItem indexDataItem; foreach (IndexItem queryIndexItem in containsIndexQuery.IndexItemList) { #region Search item in index searchIndex = internalCacheIndexList[0].Search(queryIndexItem); #endregion if (searchIndex > -1) { if (multiItemResult == null) { multiItemResult = new MultiItemResult(containsIndexQuery.IndexId); } indexDataItem = new IndexDataItem(InternalItemAdapter.ConvertToIndexItem(internalCacheIndexList[0].GetItem(searchIndex), internalCacheIndexList[0].InDeserializationContext)); #region Get extra tags if (containsIndexQuery.TagsFromIndexes != null && containsIndexQuery.TagsFromIndexes.Count != 0) { foreach (string indexName in containsIndexQuery.TagsFromIndexes) { Index indexInfo = indexTypeMapping.IndexCollection[indexName]; CacheIndexInternal indexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, containsIndexQuery.IndexId, indexInfo.ExtendedIdSuffix, indexName, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null, indexInfo.IsMetadataPropertyCollection, null, containsIndexQuery.DomainSpecificProcessingType, storeContext.DomainSpecificConfig, null, null, false); if (indexInternal != null) { // update the performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery, messageContext.TypeId, indexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery, messageContext.TypeId, indexInternal.OutDeserializationContext.ReadItemCount); internalCacheIndexList.Add(indexInternal); IndexServerUtils.GetTags(indexInternal, queryIndexItem, indexDataItem); } } } #endregion multiItemResult.Add(indexDataItem); } } #region Get data if (!containsIndexQuery.ExcludeData && multiItemResult != null) { byte[] extendedId; List <RelayMessage> dataStoreMessages = new List <RelayMessage>(multiItemResult.Count); short relatedTypeId; if (containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null) { if (!storeContext.TryGetTypeId(containsIndexQuery.FullDataIdInfo.RelatedTypeName, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", containsIndexQuery.FullDataIdInfo.RelatedTypeName); throw new Exception("Invalid RelatedTypeId for TypeId - " + containsIndexQuery.FullDataIdInfo.RelatedTypeName); } } else if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } foreach (IndexDataItem resultItem in multiItemResult) { extendedId = DataTierUtil.GetFullDataId(containsIndexQuery.IndexId, resultItem, containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null ? containsIndexQuery.FullDataIdInfo.FullDataIdFieldList : indexTypeMapping.FullDataIdFieldList); dataStoreMessages.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(extendedId), extendedId, MessageType.Get)); } storeContext.ForwarderComponent.HandleMessages(dataStoreMessages); int i = 0; foreach (IndexDataItem resultItem in multiItemResult) { if (dataStoreMessages[i].Payload != null) { resultItem.Data = dataStoreMessages[i].Payload.ByteArray; } i++; } } #endregion #region Get metadata if (containsIndexQuery.GetMetadata) { if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, containsIndexQuery.IndexId, storeContext, out metadata, out metadataPropertyCollection); } else { IndexServerUtils.GetMetadataStoredWithIndex(indexTypeMapping, internalCacheIndexList, out metadata, out metadataPropertyCollection); } } #endregion } containsIndexQueryResult = new ContainsIndexQueryResult(multiItemResult, metadata, metadataPropertyCollection, indexSize, indexExists, virtualCount, indexCap, null); } catch (Exception ex) { containsIndexQueryResult = new ContainsIndexQueryResult(null, null, null, -1, false, -1, 0, ex.Message); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing ContainsIndexQuery : {1}", messageContext.TypeId, ex); } return(containsIndexQueryResult); }
/// <summary> /// Processes the specified MetadataPropertyCommand. /// </summary> /// <param name="metadataPropertyCommand">The MetadataPropertyCommand.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MetadataPropertyCommand metadataPropertyCommand, MessageContext messageContext, IndexStoreContext storeContext) { if (metadataPropertyCommand != null) { MetadataPropertyCollection metadataPropertyCollection = null; byte[] metadata; IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; CacheIndexInternal cacheIndexInternal = null; #region Fetch MetadataPropertyCollection if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, metadataPropertyCommand.PrimaryId, metadataPropertyCommand.IndexId, storeContext, out metadata, out metadataPropertyCollection); } else { Index indexInfo = indexTypeMapping.IndexCollection[metadataPropertyCommand.TargetIndexName]; // Get CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, metadataPropertyCommand.PrimaryId, metadataPropertyCommand.IndexId, indexInfo.ExtendedIdSuffix, metadataPropertyCommand.TargetIndexName, 0, null, false, null, true, false, null, null, null, null, true, null, DomainSpecificProcessingType.None, null, null, null, true); if (cacheIndexInternal != null && cacheIndexInternal.MetadataPropertyCollection != null) { metadataPropertyCollection = cacheIndexInternal.MetadataPropertyCollection; } } #endregion #region Process MetadataPropertyCollection add/deletes if (metadataPropertyCollection == null) { metadataPropertyCollection = new MetadataPropertyCollection(); } metadataPropertyCollection.Process(metadataPropertyCommand.MetadataPropertyCollectionUpdate); #endregion #region Restore MetadataPropertyCollection back to storage bool isCompress = storeContext.GetCompressOption(messageContext.TypeId); byte[] extId = null; byte[] byteArray = null; if (indexTypeMapping.MetadataStoredSeperately) { extId = metadataPropertyCommand.ExtendedId; byteArray = Serializer.Serialize(metadataPropertyCollection, isCompress); } else { if (cacheIndexInternal == null) { cacheIndexInternal = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext(0, metadataPropertyCommand.TargetIndexName, metadataPropertyCommand.IndexId, messageContext.TypeId, null, false, null, true, false, null, null, null, null, null, null, true, null, DomainSpecificProcessingType.None, null, null, null) }; } //Restore CacheIndexInternal cacheIndexInternal.MetadataPropertyCollection = metadataPropertyCollection; extId = IndexServerUtils.FormExtendedId(metadataPropertyCommand.IndexId, indexTypeMapping.IndexCollection[ cacheIndexInternal.InDeserializationContext.IndexName]. ExtendedIdSuffix); byteArray = Serializer.Serialize(cacheIndexInternal, isCompress); } PayloadStorage bdbEntryHeader = new PayloadStorage { Compressed = isCompress, TTL = -1, LastUpdatedTicks = DateTime.Now.Ticks, ExpirationTicks = -1, Deactivated = false }; BinaryStorageAdapter.Save( storeContext.MemoryPool, storeContext.IndexStorageComponent, messageContext.TypeId, metadataPropertyCommand.PrimaryId, extId, bdbEntryHeader, byteArray); #endregion } }
/// <summary> /// Processes the specified get range query. /// </summary> /// <param name="getRangeQuery">The get range query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>GetRangeQueryResult</returns> internal static GetRangeQueryResult Process(GetRangeQuery getRangeQuery, MessageContext messageContext, IndexStoreContext storeContext) { GetRangeQueryResult getRangeQueryResult; List <ResultItem> resultItemList = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; byte[] metadata = null; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Validate Query ValidateQuery(indexTypeMapping, getRangeQuery); #endregion int maxItemsPerIndex = getRangeQuery.Offset - 1 + getRangeQuery.ItemNum; Index targetIndexInfo = indexTypeMapping.IndexCollection[getRangeQuery.TargetIndexName]; #region Prepare Result #region Extract index and apply criteria CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, getRangeQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, getRangeQuery.TargetIndexName, maxItemsPerIndex, getRangeQuery.Filter, true, null, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null); #endregion if (targetIndex != null) { indexSize = targetIndex.OutDeserializationContext.TotalCount; virtualCount = targetIndex.VirtualCount; indexExists = true; // update perf counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerGetRangeQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerGetRangeQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); #region Populate resultLists resultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, getRangeQuery.Offset, getRangeQuery.ItemNum); #endregion #region Get data if (!getRangeQuery.ExcludeData) { DataTierUtil.GetData(resultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, getRangeQuery.FullDataIdInfo); } #endregion #region Get metadata if (getRangeQuery.GetMetadata) { metadata = IndexServerUtils.GetQueryMetadata(new List <CacheIndexInternal>(1) { targetIndex }, indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, getRangeQuery.IndexId, storeContext); } #endregion #endregion } getRangeQueryResult = new GetRangeQueryResult(indexExists, indexSize, metadata, resultItemList, virtualCount, null); } catch (Exception ex) { getRangeQueryResult = new GetRangeQueryResult(false, -1, null, null, -1, ex.Message); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing GetRangeQuery : {1}", messageContext.TypeId, ex); } return(getRangeQueryResult); }
/// <summary> /// Processes the specified filtered index delete command. /// </summary> /// <param name="filteredIndexDeleteCommand">The filtered index delete command.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process( FilteredIndexDeleteCommand filteredIndexDeleteCommand, MessageContext messageContext, IndexStoreContext storeContext) { if (filteredIndexDeleteCommand != null) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; Index indexInfo = indexTypeMapping.IndexCollection[filteredIndexDeleteCommand.TargetIndexName]; #region Get CacheIndexInternal for TargetIndexName CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, filteredIndexDeleteCommand.IndexId, indexInfo.ExtendedIdSuffix, filteredIndexDeleteCommand.TargetIndexName, 0, filteredIndexDeleteCommand.DeleteFilter, false, null, false, true, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); #endregion if (cacheIndexInternal != null) { #region Increment perf counters' number PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsFilteredPerFilterDeleteRequest, messageContext.TypeId, (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count)); #endregion #region Update VirtualCount cacheIndexInternal.VirtualCount -= (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.Count); #endregion #region Save CacheIndexInternal to local storage since item which pass delete filter are pruned in it byte[] extendedId = IndexServerUtils.FormExtendedId(filteredIndexDeleteCommand.IndexId, indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix); RelayMessage indexStorageMessage = RelayMessage.GetSaveMessageForObject(messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, extendedId, DateTime.Now, cacheIndexInternal, storeContext.GetCompressOption(messageContext.TypeId)); storeContext.IndexStorageComponent.HandleMessage(indexStorageMessage); #endregion #region Data store deletes if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode) && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList != null && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count > 0) { List <RelayMessage> dataStorageMessageList = new List <RelayMessage>(); short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } cacheIndexInternal.InternalItemList = cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList; List <byte[]> fullDataIdList = DataTierUtil.GetFullDataIds(cacheIndexInternal.InDeserializationContext.IndexId, cacheIndexInternal.InternalItemList, indexTypeMapping.FullDataIdFieldList); foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } } #endregion } } }
/// <summary> /// Processes the specified DistinctQuery. /// </summary> /// <param name="distinctQuery">The DistinctQuery.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>DistinctQueryResult</returns> internal static DistinctQueryResult Process(DistinctQuery distinctQuery, MessageContext messageContext, IndexStoreContext storeContext) { DistinctQueryResult distinctQueryResult; Dictionary <byte[], int> distinctValueCountMapping = null; bool indexExists = false; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; Index targetIndexInfo = indexTypeMapping.IndexCollection[distinctQuery.TargetIndexName]; #region Prepare Result CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, distinctQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, distinctQuery.TargetIndexName, (distinctQuery.ItemsToLookUp != null ? (int)distinctQuery.ItemsToLookUp : Int32.MaxValue), null, true, distinctQuery.IndexCondition, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null, targetIndexInfo.IsMetadataPropertyCollection, null, DomainSpecificProcessingType.None, storeContext.DomainSpecificConfig, distinctQuery.FieldName, null, true); #endregion if (targetIndex != null) { indexExists = true; // update perf counter PerformanceCounters.Instance.SetCounterValue(PerformanceCounterEnum.NumOfItemsInIndexPerDistinctQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue(PerformanceCounterEnum.NumOfItemsReadPerDistinctQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); distinctValueCountMapping = targetIndex.OutDeserializationContext.DistinctValueCountMapping; } distinctQueryResult = new DistinctQueryResult { DistinctValueCountMapping = distinctValueCountMapping, IndexExists = indexExists, }; } catch (Exception ex) { distinctQueryResult = new DistinctQueryResult { DistinctValueCountMapping = null, IndexExists = false, ExceptionInfo = ex.Message }; LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing DistinctQuery : {1}", messageContext.TypeId, ex); } return(distinctQueryResult); }