/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MessageContext messageContext, IndexStoreContext storeContext) { // TBD : Support concurrent DeleteAllInType for different types lock (LockingUtil.Instance.LockerObjects) { RelayMessage msg; if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId].IndexServerMode)) { // Send DeleteAll to Data Store short relatedTypeId; if(!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } msg = new RelayMessage(relatedTypeId, 0, MessageType.DeleteAllInType); storeContext.ForwarderComponent.HandleMessage(msg); } // Send DeleteAll to local Index storage msg = new RelayMessage(messageContext.TypeId, 0, MessageType.DeleteAllInType); storeContext.IndexStorageComponent.HandleMessage(msg); // Delete hash mapping entries in file storeContext.RemoveType(messageContext.TypeId); } }
/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>Raw CacheIndexInternal</returns> internal static byte[] Process(MessageContext messageContext, IndexStoreContext storeContext) { byte[] payloadByteArray = null; RelayMessage getMsg = new RelayMessage { MessageType = MessageType.Get, Id = IndexCacheUtils.GeneratePrimaryId(messageContext.ExtendedId), ExtendedId = IndexServerUtils.FormExtendedId(messageContext.ExtendedId, 0) // Pull first of the multiple indexes }; if (storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection.Contains(messageContext.TypeId)) { getMsg.TypeId = messageContext.TypeId; } else if (storeContext.RelatedTypeIds.TryGetValue(messageContext.TypeId, out getMsg.TypeId)) { } else { LoggingUtil.Log.InfoFormat("Invalid TypeID for GetMessage {0}", messageContext.TypeId); return payloadByteArray; } storeContext.IndexStorageComponent.HandleMessage(getMsg); if (getMsg.Payload != null) { payloadByteArray = getMsg.Payload.ByteArray; } return payloadByteArray; }
/// <summary> /// Gets the data into resultItemList items. /// </summary> /// <param name="resultItemList">The result item list.</param> /// <param name="storeContext">The store context.</param> /// <param name="messageContext">The message context.</param> /// <param name="fullDataIdFieldList">The full data id field list.</param> /// <param name="fullDataIdInfo">The full data id info.</param> internal static void GetData(List<ResultItem> resultItemList, IndexStoreContext storeContext, MessageContext messageContext, FullDataIdFieldList fullDataIdFieldList, FullDataIdInfo fullDataIdInfo) { byte[] extendedId; List<RelayMessage> dataStoreMessages = new List<RelayMessage>(); short relatedTypeId; if (fullDataIdInfo != null && fullDataIdInfo.RelatedTypeName != null) { if (!storeContext.TryGetTypeId(fullDataIdInfo.RelatedTypeName, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", fullDataIdInfo.RelatedTypeName); throw new Exception("Invalid RelatedTypeId for TypeId - " + fullDataIdInfo.RelatedTypeName); } } else if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } if (resultItemList != null && resultItemList.Count > 0) { foreach (ResultItem resultItem in resultItemList) { extendedId = GetFullDataId(resultItem.IndexId, resultItem, fullDataIdInfo != null && fullDataIdInfo.RelatedTypeName != null ? fullDataIdInfo.FullDataIdFieldList : fullDataIdFieldList); dataStoreMessages.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(extendedId), extendedId, MessageType.Get)); } storeContext.ForwarderComponent.HandleMessages(dataStoreMessages); int i = 0; foreach (ResultItem resultItem in resultItemList) { if (dataStoreMessages[i].Payload != null) { resultItem.Data = dataStoreMessages[i].Payload.ByteArray; } else { LoggingUtil.Log.DebugFormat("Fetched Null Data for TypeId: {0}, IndexId: {1}, ItemId: {2}, FullDataId: {3}, PrimaryId: {4}", relatedTypeId, IndexCacheUtils.GetReadableByteArray(resultItem.IndexId), IndexCacheUtils.GetReadableByteArray(resultItem.ItemId), IndexCacheUtils.GetReadableByteArray(dataStoreMessages[i].ExtendedId), IndexCacheUtils.GeneratePrimaryId(dataStoreMessages[i].ExtendedId) ); } i++; } } }
public void GetDataItems(FullDataIdInfo info, bool excludeData, MessageContext messageContext, IndexStoreContext storeContext, BaseMultiIndexIdQueryResult queryResult) { if (excludeData == false) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; DataTierUtil.GetData(queryResult.ResultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, info); } }
/// <summary> /// Processes the specified cache index update. /// </summary> /// <param name="cacheIndexUpdate">The cache index update.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(CacheIndexUpdate cacheIndexUpdate, MessageContext messageContext, IndexStoreContext storeContext) { switch (cacheIndexUpdate.Command.CommandType) { case CommandType.FilteredIndexDelete: PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.FilterDelete, messageContext.TypeId, 1); FilteredIndexDeleteProcessor.Process(cacheIndexUpdate.Command as FilteredIndexDeleteCommand, messageContext, storeContext); break; } }
/// <summary> /// Processes the specified remote clustered intersection query. /// </summary> /// <param name="remoteClusteredIntersectionQuery">The remote clustered intersection query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>IntersectionQueryResult</returns> internal static IntersectionQueryResult Process(RemoteClusteredIntersectionQuery remoteClusteredIntersectionQuery, MessageContext messageContext, IndexStoreContext storeContext) { // increment performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.IndexLookupAvgPerRemoteClusteredIntersectionQuery, messageContext.TypeId, remoteClusteredIntersectionQuery.IndexIdList.Count); VirtualClusteredIntersectionQuery query = new VirtualClusteredIntersectionQuery( remoteClusteredIntersectionQuery.IndexIdList, remoteClusteredIntersectionQuery.TargetIndexName, storeContext.GetTypeName(messageContext.TypeId)) { ExcludeData = remoteClusteredIntersectionQuery.ExcludeData, GetIndexHeader = remoteClusteredIntersectionQuery.GetIndexHeader, Filter = remoteClusteredIntersectionQuery.Filter, intersectionQueryParamsMapping = remoteClusteredIntersectionQuery.IntersectionQueryParamsMapping, PrimaryId = remoteClusteredIntersectionQuery.PrimaryId, PrimaryIdList = remoteClusteredIntersectionQuery.PrimaryIdList }; return RelayClient.Instance.SubmitQuery<VirtualClusteredIntersectionQuery, IntersectionQueryResult>(query); }
internal SpanQueryResult Process(RemoteClusteredSpanQuery remoteClusteredSpanQuery, MessageContext messageContext, IndexStoreContext storeContext) { // increment the performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.IndexLookupAvgPerRemoteClusteredSpanQuery, messageContext.TypeId, remoteClusteredSpanQuery.IndexIdList.Count); // creating a spanQuery with copy ctor VirtualSpanQuery query = new VirtualSpanQuery(remoteClusteredSpanQuery) { CacheTypeName = storeContext.GetTypeName(messageContext.TypeId), // we get only reference at this point ExcludeData = true }; SpanQueryResult queryResult = RelayClient.Instance.SubmitQuery<VirtualSpanQuery, SpanQueryResult>(query); GetDataItems(remoteClusteredSpanQuery.FullDataIdInfo, remoteClusteredSpanQuery.ExcludeData, messageContext, storeContext, queryResult); return queryResult; }
/// <summary> /// Processes the get message. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessGetMessage(RelayMessage message, MessageContext messageContext) { byte[] payloadByteArray = GetProcessor.Process(messageContext, storeContext); if (payloadByteArray != null) { bool compress = storeContext.GetCompressOption(message.TypeId); message.Payload = new RelayPayload(message.TypeId, message.Id, payloadByteArray, compress); } else { LoggingUtil.Log.InfoFormat("Miss in CacheIndexStorage for Id : {0}, ExtendedId : ", message.Id, IndexCacheUtils.GetReadableByteArray(message.ExtendedId)); } }
/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MessageContext messageContext, IndexStoreContext storeContext) { lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId)) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; List<RelayMessage> dataStorageMessageList = new List<RelayMessage>(); List<RelayMessage> indexStorageMessageList = new List<RelayMessage>(indexTypeMapping.IndexCollection.Count); CacheIndexInternal internalIndex; List<byte[]> fullDataIdList; if (indexTypeMapping.MetadataStoredSeperately) { indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId, messageContext.PrimaryId, messageContext.ExtendedId, MessageType.Delete)); } foreach (Index index in indexTypeMapping.IndexCollection) { internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, messageContext.ExtendedId, index.ExtendedIdSuffix, index.IndexName, 0, null, true, null, false, false, index.PrimarySortInfo, index.LocalIdentityTagList, index.StringHashCodeDictionary, null); if (internalIndex != null) { #region Deletes messages for data store if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode)) { fullDataIdList = DataTierUtil.GetFullDataIds(messageContext.ExtendedId, internalIndex.InternalItemList, indexTypeMapping.FullDataIdFieldList); short relatedTypeId; foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { if (storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } else { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } } } } #endregion #region Delete messages for index store indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId, messageContext.PrimaryId, IndexServerUtils.FormExtendedId(messageContext.ExtendedId, index.ExtendedIdSuffix), MessageType.Delete)); #endregion } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } if (indexStorageMessageList.Count > 0) { storeContext.IndexStorageComponent.HandleMessages(indexStorageMessageList); } } }
/// <summary> /// Processes the specified filtered index delete command. /// </summary> /// <param name="filteredIndexDeleteCommand">The filtered index delete command.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process( FilteredIndexDeleteCommand filteredIndexDeleteCommand, MessageContext messageContext, IndexStoreContext storeContext) { if (filteredIndexDeleteCommand != null) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; Index indexInfo = indexTypeMapping.IndexCollection[filteredIndexDeleteCommand.TargetIndexName]; #region Get CacheIndexInternal for TargetIndexName CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, filteredIndexDeleteCommand.IndexId, indexInfo.ExtendedIdSuffix, filteredIndexDeleteCommand.TargetIndexName, 0, filteredIndexDeleteCommand.DeleteFilter, false, null, false, true, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); #endregion if (cacheIndexInternal != null) { #region Increment perf counters' number PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsFilteredPerFilterDeleteRequest, messageContext.TypeId, (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count)); #endregion #region Update VirtualCount cacheIndexInternal.VirtualCount -= (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.Count); #endregion #region Save CacheIndexInternal to local storage since item which pass delete filter are pruned in it byte[] extendedId = IndexServerUtils.FormExtendedId(filteredIndexDeleteCommand.IndexId, indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix); RelayMessage indexStorageMessage = RelayMessage.GetSaveMessageForObject(messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, extendedId, DateTime.Now, cacheIndexInternal, storeContext.GetCompressOption(messageContext.TypeId)); storeContext.IndexStorageComponent.HandleMessage(indexStorageMessage); #endregion #region Data store deletes if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode) && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList != null && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count > 0) { List<RelayMessage> dataStorageMessageList = new List<RelayMessage>(); short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } cacheIndexInternal.InternalItemList = cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList; List<byte[]> fullDataIdList = DataTierUtil.GetFullDataIds(cacheIndexInternal.InDeserializationContext.IndexId, cacheIndexInternal.InternalItemList, indexTypeMapping.FullDataIdFieldList); foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } } #endregion } } }
/// <summary> /// Processes the delete message. /// </summary> /// <param name="messageContext">The message context.</param> private void ProcessDeleteMessage(MessageContext messageContext) { DeleteProcessor.Process(messageContext, storeContext); }
/// <summary> /// Processes the specified contains index query. /// </summary> /// <param name="containsIndexQuery">The contains index query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>ContainsIndexQueryResult</returns> internal static ContainsIndexQueryResult Process(ContainsIndexQuery containsIndexQuery, MessageContext messageContext, IndexStoreContext storeContext) { ContainsIndexQueryResult containsIndexQueryResult; MultiItemResult multiItemResult = null; byte[] metadata = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Check TargetIndexName if (!indexTypeMapping.IndexCollection.Contains(containsIndexQuery.TargetIndexName)) { throw new Exception("Invalid TargetIndexName - " + containsIndexQuery.TargetIndexName); } #endregion Index targetIndexInfo = indexTypeMapping.IndexCollection[containsIndexQuery.TargetIndexName]; List<CacheIndexInternal> internalCacheIndexList = new List<CacheIndexInternal>(); #region Get TargetIndex CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, containsIndexQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, containsIndexQuery.TargetIndexName, 0, null, true, null, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null); #endregion if (cacheIndexInternal != null) { internalCacheIndexList.Add(cacheIndexInternal); indexExists = true; indexSize = cacheIndexInternal.OutDeserializationContext.TotalCount; virtualCount = cacheIndexInternal.VirtualCount; // update the performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); int searchIndex; IndexDataItem indexDataItem; foreach (IndexItem queryIndexItem in containsIndexQuery.IndexItemList) { #region Search item in index searchIndex = internalCacheIndexList[0].Search(queryIndexItem); #endregion if (searchIndex > -1) { if (multiItemResult == null) { multiItemResult = new MultiItemResult(containsIndexQuery.IndexId); } indexDataItem = new IndexDataItem(InternalItemAdapter.ConvertToIndexItem(internalCacheIndexList[0].GetItem(searchIndex), internalCacheIndexList[0].InDeserializationContext)); #region Get extra tags if (containsIndexQuery.TagsFromIndexes != null && containsIndexQuery.TagsFromIndexes.Count != 0) { foreach (string indexName in containsIndexQuery.TagsFromIndexes) { Index indexInfo = indexTypeMapping.IndexCollection[indexName]; CacheIndexInternal indexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, containsIndexQuery.IndexId, indexInfo.ExtendedIdSuffix, indexName, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (indexInternal != null) { // update the performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery, messageContext.TypeId, indexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery, messageContext.TypeId, indexInternal.OutDeserializationContext.ReadItemCount); internalCacheIndexList.Add(indexInternal); IndexServerUtils.GetTags(indexInternal, queryIndexItem, indexDataItem); } } } #endregion multiItemResult.Add(indexDataItem); } } #region Get data if (!containsIndexQuery.ExcludeData && multiItemResult != null) { byte[] extendedId; List<RelayMessage> dataStoreMessages = new List<RelayMessage>(multiItemResult.Count); short relatedTypeId; if (containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null) { if (!storeContext.TryGetTypeId(containsIndexQuery.FullDataIdInfo.RelatedTypeName, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", containsIndexQuery.FullDataIdInfo.RelatedTypeName); throw new Exception("Invalid RelatedTypeId for TypeId - " + containsIndexQuery.FullDataIdInfo.RelatedTypeName); } } else if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } foreach (IndexDataItem resultItem in multiItemResult) { extendedId = DataTierUtil.GetFullDataId(containsIndexQuery.IndexId, resultItem, containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null ? containsIndexQuery.FullDataIdInfo.FullDataIdFieldList : indexTypeMapping.FullDataIdFieldList); dataStoreMessages.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(extendedId), extendedId, MessageType.Get)); } storeContext.ForwarderComponent.HandleMessages(dataStoreMessages); int i = 0; foreach (IndexDataItem resultItem in multiItemResult) { if (dataStoreMessages[i].Payload != null) { resultItem.Data = dataStoreMessages[i].Payload.ByteArray; } i++; } } #endregion #region Get metadata if (containsIndexQuery.GetMetadata) { metadata = IndexServerUtils.GetQueryMetadata(internalCacheIndexList, indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, containsIndexQuery.IndexId, storeContext); } #endregion } containsIndexQueryResult = new ContainsIndexQueryResult(multiItemResult, metadata, indexSize, indexExists, virtualCount, null); } catch (Exception ex) { containsIndexQueryResult = new ContainsIndexQueryResult(null, null, -1, false, -1, ex.Message); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing ContainsIndexQuery : {1}", messageContext.TypeId, ex); } return containsIndexQueryResult; }
/// <summary> /// Handles the message. /// </summary> /// <param name="message">The message.</param> public void HandleMessage(RelayMessage message) { MessageContext messageContext = new MessageContext { PrimaryId = message.Id, TypeId = message.TypeId, ExtendedId = message.ExtendedId, RelayTTL = message.RelayTTL, SourceZone = message.SourceZone, AddressHistory = message.AddressHistory }; switch (message.MessageType) { case MessageType.Query: ProcessQueryMessage(message, messageContext); break; case MessageType.Save: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.Save, messageContext.TypeId, 1); ProcessSaveMessage(message, messageContext); break; case MessageType.Update: ProcessUpdateMessage(message, messageContext); break; case MessageType.Delete: ProcessDeleteMessage(messageContext); break; case MessageType.DeleteAll: LoggingUtil.Log.InfoFormat("CacheIndexV3Store does NOT support Message Type {0}", message.MessageType); break; case MessageType.DeleteAllInType: ProcessDeleteAllInType(messageContext); break; case MessageType.DeleteInAllTypes: LoggingUtil.Log.InfoFormat("CacheIndexV3Store does NOT support Message Type {0}", message.MessageType); // do not throw exception - we can safely ignore this message type break; case MessageType.Get: ProcessGetMessage(message, messageContext); break; default: LoggingUtil.Log.ErrorFormat("Unknown Message Type {0}", message.MessageType); break; } }
/// <summary> /// Processes the query message. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessQueryMessage(RelayMessage message, MessageContext messageContext) { switch (message.QueryId) { case (byte)QueryTypes.PagedTaggedIndexQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.PagedIndexQuery, messageContext.TypeId, 1); ProcessPagedIndexQuery(message, messageContext); break; case (byte)QueryTypes.RemoteClusteredPagedIndexQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.RemoteClusteredPagedIndexQuery, messageContext.TypeId, 1); ProcessRemoteClusteredPagedIndexQuery(message, messageContext); break; case (byte)QueryTypes.ContainsIndexQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.ContainsIndexQuery, messageContext.TypeId, 1); ProcessContainsIndexQuery(message, messageContext); break; case (byte)QueryTypes.FirstLastQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.FirstLastQuery, messageContext.TypeId, 1); ProcessFirstLastQuery(message, messageContext); break; case (byte)QueryTypes.GetRangeQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.GetRangeQuery, messageContext.TypeId, 1); ProcessGetRangeQuery(message, messageContext); break; case (byte)QueryTypes.RandomQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.RandomQuery, messageContext.TypeId, 1); ProcessRandomQuery(message, messageContext); break; case (byte)QueryTypes.IntersectionQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.IntersectionQuery, messageContext.TypeId, 1); ProcessIntersectionQuery(message, messageContext); break; case (byte)QueryTypes.RemoteClusteredIntersectionQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.RemoteClusteredIntersectionQuery, messageContext.TypeId, 1); ProcessRemoteClusteredIntersectionQuery(message, messageContext); break; case (byte)QueryTypes.SpanQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.SpanQuery, messageContext.TypeId, 1); ProcessSpanQuery(message, messageContext); break; case (byte)QueryTypes.RemoteClusteredSpanQuery: // increment performance counter PerformanceCounters.Instance.IncrementCounter( PerformanceCounterEnum.RemoteClusteredSpanQuery, messageContext.TypeId, 1); ProcessRemoteClusteredSpanQuery(message, messageContext); break; default: throw new Exception("QueryId +" + (int)message.QueryId + "NOT supported"); } }
/// <summary> /// Processes the specified random query. /// </summary> /// <param name="randomQuery">The random query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>RandomQueryResult</returns> internal static RandomQueryResult Process(RandomQuery randomQuery, MessageContext messageContext, IndexStoreContext storeContext) { RandomQueryResult randomQueryResult; List<ResultItem> resultItemList = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; byte[] metadata = null; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Validate Query ValidateQuery(indexTypeMapping, randomQuery); #endregion Index targetIndexInfo = indexTypeMapping.IndexCollection[randomQuery.TargetIndexName]; #region Prepare Result #region Extract index and apply criteria CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, randomQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, randomQuery.TargetIndexName, 0, randomQuery.Filter, true, null, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null); #endregion if (targetIndex != null) { indexSize = targetIndex.OutDeserializationContext.TotalCount; virtualCount = targetIndex.VirtualCount; indexExists = true; // update performance counters PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerRandomQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerRandomQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); #region Populate resultLists int indexItemCount = targetIndex.Count; int resultListCount = Math.Min(randomQuery.Count, indexItemCount); IEnumerable<int> itemPositionList = Algorithm.RandomSubset(new Random(), 0, indexItemCount - 1, resultListCount); resultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, itemPositionList); #endregion #region Get data if (!randomQuery.ExcludeData) { DataTierUtil.GetData(resultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, randomQuery.FullDataIdInfo); } #endregion #region Get metadata if (randomQuery.GetMetadata) { metadata = IndexServerUtils.GetQueryMetadata(new List<CacheIndexInternal>(1) { targetIndex }, indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, randomQuery.IndexId, storeContext); } #endregion #endregion } randomQueryResult = new RandomQueryResult(indexExists, indexSize, metadata, resultItemList, virtualCount, null); } catch (Exception ex) { randomQueryResult = new RandomQueryResult(false, -1, null, null, -1, ex.Message); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing RandomQuery : {1}", messageContext.TypeId, ex); } return randomQueryResult; }
/// <summary> /// Processes the update message. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessUpdateMessage(RelayMessage message, MessageContext messageContext) { lock (LockingUtil.Instance.GetLock(message.Id)) { try { CacheIndexUpdate cacheIndexUpdate = message.GetObject<CacheIndexUpdate>(); UpdateProcessor.Process(cacheIndexUpdate, messageContext, storeContext); } catch (Exception ex) { LoggingUtil.Log.ErrorFormat("TypeID {0} -- Error processing update message : {1}", message.TypeId, ex); throw new Exception("Error processing update message"); } } }
/// <summary> /// Processes the contains index query. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessContainsIndexQuery(RelayMessage message, MessageContext messageContext) { ContainsIndexQueryResult containsIndexQueryResult = ContainsQueryProcessor.Process(message.GetQueryObject<ContainsIndexQuery>(), messageContext, storeContext); bool compressOption = storeContext.GetCompressOption(message.TypeId); message.Payload = new RelayPayload(message.TypeId, message.Id, Serializer.Serialize(containsIndexQueryResult, compressOption), compressOption); }
/// <summary> /// Processes the first last query. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessFirstLastQuery(RelayMessage message, MessageContext messageContext) { FirstLastQueryResult firstLastQueryResult = FirstLastQueryProcessor.Process(message.GetQueryObject<FirstLastQuery>(), messageContext, storeContext); bool compressOption = storeContext.GetCompressOption(message.TypeId); message.Payload = new RelayPayload(message.TypeId, message.Id, Serializer.Serialize(firstLastQueryResult, compressOption), compressOption); }
/// <summary> /// Processes the remote clustered intersection query. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessRemoteClusteredIntersectionQuery(RelayMessage message, MessageContext messageContext) { IntersectionQueryResult intersectionQueryResult = RemoteClusteredIntersectionQueryProcessor.Process(message.GetQueryObject<RemoteClusteredIntersectionQuery>(), messageContext, storeContext); AttachIntersectionQueryPayload(message, intersectionQueryResult); }
/// <summary> /// Processes the specified cache index. /// </summary> /// <param name="cacheIndex">Index of the cache.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(CacheIndex cacheIndex, MessageContext messageContext, IndexStoreContext storeContext) { lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId)) { try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Extract CacheIndex and Validate from incoming message ValidateSave(cacheIndex); #endregion #region Log CacheIndex before processing StringBuilder dbgIndexInfo = null; if (LoggingUtil.Log.IsDebugEnabled) { dbgIndexInfo = new StringBuilder(); dbgIndexInfo.Append("TypeId=").Append(messageContext.TypeId).Append(Environment.NewLine); dbgIndexInfo.Append(IndexServerUtils.GetPrintableCacheIndex(cacheIndex, storeContext.TagHashCollection, messageContext.TypeId)); } #endregion #region Init vars List<RelayMessage> indexStorageMessageList = new List<RelayMessage>(); List<RelayMessage> dataStorageMessageList = new List<RelayMessage>(); List<CacheIndexInternal> internalIndexList = new List<CacheIndexInternal>(); List<IndexItem> cappedDeleteItemList = new List<IndexItem>(); CacheIndexInternal internalIndex; #endregion if (cacheIndex.IndexVirtualCountMapping == null) { #region Save Items #region Extract CacheIndexInternal from index storage if (cacheIndex.TargetIndexName == null) //Save to multiple indexes { #region Get CacheIndexInternal for multiple indexes foreach (KeyValuePair<string /*IndexName*/, List<string> /*TagNameList*/> kvp in cacheIndex.IndexTagMapping) { Index indexInfo = indexTypeMapping.IndexCollection[kvp.Key]; internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, indexInfo.ExtendedIdSuffix, kvp.Key, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (internalIndex != null) { // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumberOfItemsInIndexPerSave, messageContext.TypeId, internalIndex.OutDeserializationContext.TotalCount); } if (internalIndex == null || cacheIndex.ReplaceFullIndex) //CacheIndexInternal does not exists or is to be discarded { internalIndex = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext { TypeId = messageContext.TypeId, TagHashCollection = storeContext.TagHashCollection, IndexId = cacheIndex.IndexId, IndexName = kvp.Key, InclusiveFilter = true, PrimarySortInfo = indexInfo.PrimarySortInfo, LocalIdentityTagNames = indexInfo.LocalIdentityTagList, StringHashCollection = storeContext.StringHashCollection, StringHashCodeDictionary = indexInfo.StringHashCodeDictionary } }; } internalIndexList.Add(internalIndex); } #endregion } else //Save to single index { #region Get CacheIndexInternal for TargetIndexName Index indexInfo = indexTypeMapping.IndexCollection[cacheIndex.TargetIndexName]; internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, indexInfo.ExtendedIdSuffix, cacheIndex.TargetIndexName, 0, null, true, null, false, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (internalIndex != null) { // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumberOfItemsInIndexPerSave, messageContext.TypeId, internalIndex.OutDeserializationContext.TotalCount); } if (internalIndex == null || cacheIndex.ReplaceFullIndex) //CacheIndexInternal does not exists or is to be discarded { internalIndex = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext { TypeId = messageContext.TypeId, TagHashCollection = storeContext.TagHashCollection, IndexId = cacheIndex.IndexId, IndexName = cacheIndex.TargetIndexName, InclusiveFilter = true, PrimarySortInfo = indexInfo.PrimarySortInfo, LocalIdentityTagNames = indexInfo.LocalIdentityTagList, StringHashCollection = storeContext.StringHashCollection, StringHashCodeDictionary = indexInfo.StringHashCodeDictionary } }; } internalIndexList.Add(internalIndex); #endregion } #endregion #region Log CacheIndexInternals before save if (LoggingUtil.Log.IsDebugEnabled && dbgIndexInfo != null) { dbgIndexInfo.Append(Environment.NewLine).Append(string.Format("BEFORE SAVE {0}", IndexServerUtils.GetPrintableCacheIndexInternalList( internalIndexList, storeContext.TagHashCollection, messageContext.TypeId))); } #endregion #region Process Delete and Add List try { #region Process Delete List if (cacheIndex.DeleteList.Count > 0 && !cacheIndex.ReplaceFullIndex) { ProcessDeleteList(internalIndexList, cacheIndex.DeleteList, messageContext.TypeId); } #endregion #region Process Add List if (cacheIndex.AddList.Count > 0 || cacheIndex.UpdateMetadata) { ProcessAddList(internalIndexList, cappedDeleteItemList, cacheIndex, storeContext, indexTypeMapping); } #endregion } catch { LoggingUtil.Log.Debug(IndexServerUtils.GetPrintableCacheIndexInternalList(internalIndexList, storeContext.TagHashCollection, messageContext.TypeId)); throw; } #endregion #region Log CacheIndexInternals after save if (LoggingUtil.Log.IsDebugEnabled && dbgIndexInfo != null) { dbgIndexInfo.Append(Environment.NewLine).Append(string.Format("AFTER SAVE {0}", IndexServerUtils.GetPrintableCacheIndexInternalList(internalIndexList, storeContext.TagHashCollection, messageContext.TypeId))); } #endregion #region Data store relay messages for deletes and saves if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode) && !cacheIndex.PreserveData) { byte[] fullDataId; short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } #region Delete Messages foreach (IndexItem indexItem in cacheIndex.DeleteList) { fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexItem, indexTypeMapping.FullDataIdFieldList); if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } #endregion #region Save Messages foreach (IndexDataItem indexDataItem in cacheIndex.AddList) { fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexDataItem, indexTypeMapping.FullDataIdFieldList); if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, DateTime.Now, indexDataItem.Data ?? new byte[0], storeContext.GetCompressOption(messageContext.TypeId), MessageType.Save)); if (indexDataItem.Data == null || indexDataItem.Data.Length == 0) { LoggingUtil.Log.WarnFormat("Saving null data for TypeId: {0}, IndexId: {1}, ItemId: {2}, FullDataId: {3}, PrimaryId: {4}", relatedTypeId, IndexCacheUtils.GetReadableByteArray(cacheIndex.IndexId), IndexCacheUtils.GetReadableByteArray(indexDataItem.ItemId), IndexCacheUtils.GetReadableByteArray(fullDataId), IndexCacheUtils.GeneratePrimaryId(fullDataId)); } } } #endregion #region Capped Item Delete Messages foreach (IndexItem indexItem in cappedDeleteItemList) { fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexItem, indexTypeMapping.FullDataIdFieldList); if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } #endregion #region Send relay mesaages to data store if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } #endregion } #endregion #endregion if (dbgIndexInfo != null) { LoggingUtil.Log.Debug(dbgIndexInfo.ToString()); } } else { #region Update Virtual Count foreach (KeyValuePair<string /*IndexName*/, int /*VirtualCount*/> kvp in cacheIndex.IndexVirtualCountMapping) { Index indexInfo = indexTypeMapping.IndexCollection[kvp.Key]; internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, indexInfo.ExtendedIdSuffix, kvp.Key, 0, null, true, null, true, false, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null); if (internalIndex == null) { internalIndex = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext { TypeId = messageContext.TypeId, TagHashCollection = storeContext.TagHashCollection, IndexId = cacheIndex.IndexId, IndexName = kvp.Key, InclusiveFilter = true, DeserializeHeaderOnly = true, PrimarySortInfo = indexInfo.PrimarySortInfo, LocalIdentityTagNames = indexInfo.LocalIdentityTagList, StringHashCollection = storeContext.StringHashCollection, StringHashCodeDictionary = indexInfo.StringHashCodeDictionary } }; } else { // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumberOfItemsInIndexPerSave, messageContext.TypeId, internalIndex.OutDeserializationContext.TotalCount); } internalIndex.VirtualCount = kvp.Value; internalIndexList.Add(internalIndex); } #endregion } #region Index storage relay messages for each CacheIndexInternal #region Metadata if (indexTypeMapping.MetadataStoredSeperately && cacheIndex.UpdateMetadata) { indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId, cacheIndex.PrimaryId, cacheIndex.IndexId, DateTime.Now, cacheIndex.Metadata ?? new byte[0], storeContext.GetCompressOption(messageContext.TypeId), MessageType.Save)); } #endregion #region Index(es) byte[] payload; CompactBinaryWriter writer; RelayMessage indexStorageMessage; byte[] extendedId; foreach (CacheIndexInternal cacheIndexInternal in internalIndexList) { extendedId = IndexServerUtils.FormExtendedId( cacheIndex.IndexId, indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix); // This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal // ** TBD - Should be removed later if (LegacySerializationUtil.Instance.IsSupported(messageContext.TypeId)) { writer = new CompactBinaryWriter(new BinaryWriter(new MemoryStream())); cacheIndexInternal.Serialize(writer); payload = new byte[writer.BaseStream.Length]; writer.BaseStream.Position = 0; writer.BaseStream.Read(payload, 0, payload.Length); indexStorageMessage = new RelayMessage(messageContext.TypeId, cacheIndex.PrimaryId, extendedId, DateTime.Now, payload, storeContext.GetCompressOption(messageContext.TypeId), MessageType.Save); } else { indexStorageMessage = RelayMessage.GetSaveMessageForObject(messageContext.TypeId, cacheIndex.PrimaryId, extendedId, DateTime.Now, cacheIndexInternal, storeContext.GetCompressOption(messageContext.TypeId)); } indexStorageMessageList.Add(indexStorageMessage); } #endregion #region Send relay mesaages to index storage storeContext.IndexStorageComponent.HandleMessages(indexStorageMessageList); #endregion #endregion } catch (Exception ex) { LoggingUtil.Log.DebugFormat("CacheIndex: {0}", IndexServerUtils.GetPrintableCacheIndex(cacheIndex, storeContext.TagHashCollection, messageContext.TypeId)); throw new Exception("TypeId " + messageContext.TypeId + " -- Error processing save message.", ex); } } }
/// <summary> /// Processes the specified intersection query. /// </summary> /// <param name="intersectionQuery">The intersection query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>IntersectionQueryResult</returns> internal static IntersectionQueryResult Process(IntersectionQuery intersectionQuery, MessageContext messageContext, IndexStoreContext storeContext) { //Fetch each index (assume all indexes are local) and perform intersection and return the results IntersectionQueryResult intersectionQueryResult; List<IndexDataItem> resultItemList = null; Dictionary<byte[], IndexHeader> indexIdIndexHeaderMapping = new Dictionary<byte[], IndexHeader>(new ByteArrayEqualityComparer()); List<string> localIdentityTagNames = null; bool isTagPrimarySort = false; string sortFieldName = null; List<SortOrder> sortOrderList = null; StringBuilder exceptionInfo = new StringBuilder(); try { if (intersectionQuery.IndexIdList.Count > 0) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; ValidateQuery(indexTypeMapping, intersectionQuery); #region Set sort vars Index targetIndexInfo = indexTypeMapping.IndexCollection[intersectionQuery.TargetIndexName]; localIdentityTagNames = indexTypeMapping.IndexCollection[intersectionQuery.TargetIndexName].LocalIdentityTagList; bool sortFieldPartOfLocalId = IsSortFieldPartOfLocalId(localIdentityTagNames, targetIndexInfo.PrimarySortInfo); TagSort itemIdTagSort = new TagSort("ItemId", false, new SortOrder(DataType.Int32, SortBy.ASC)); if (!sortFieldPartOfLocalId) { //Set sort vars isTagPrimarySort = itemIdTagSort.IsTag; sortFieldName = itemIdTagSort.TagName; sortOrderList = new List<SortOrder>(1) { itemIdTagSort.SortOrder }; } else { isTagPrimarySort = targetIndexInfo.PrimarySortInfo.IsTag; sortFieldName = targetIndexInfo.PrimarySortInfo.FieldName; sortOrderList = targetIndexInfo.PrimarySortInfo.SortOrderList; } #endregion #region Fetch CacheIndexInternal and Intersect CacheIndexInternal targetIndex; CacheIndexInternal resultCacheIndexInternal = null; IntersectionQueryParams indexIdParam; byte[] indexId; for (int i = 0; i < intersectionQuery.IndexIdList.Count; i++) { #region Extract index and apply criteria indexId = intersectionQuery.IndexIdList[i]; indexIdParam = intersectionQuery.GetIntersectionQueryParamForIndexId(indexId); targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, (intersectionQuery.PrimaryIdList != null && i < intersectionQuery.PrimaryIdList.Count) ? intersectionQuery.PrimaryIdList[i] : IndexCacheUtils.GeneratePrimaryId(indexId), indexId, targetIndexInfo.ExtendedIdSuffix, intersectionQuery.TargetIndexName, 0, indexIdParam.Filter, true, null, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null); #endregion if (targetIndex != null) { if (targetIndex.Count < 0) { // No items in one of the indexes. Stop Interestion !! resultCacheIndexInternal = null; break; } PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerIntersectionQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerIntersectionQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); if (!sortFieldPartOfLocalId) { //Need to sort indexes by item id targetIndex.Sort(itemIdTagSort); } #region Intersection if (resultCacheIndexInternal == null) { // No need to perform intersection for first index resultCacheIndexInternal = targetIndex; } else { IntersectionAlgo.Intersect(isTagPrimarySort, sortFieldName, localIdentityTagNames, sortOrderList, resultCacheIndexInternal.InternalItemList, targetIndex.InternalItemList); if (resultCacheIndexInternal == null || resultCacheIndexInternal.Count < 1) { // Unable to fetch one of the indexes. Stop Interestion !! resultCacheIndexInternal = null; indexIdIndexHeaderMapping = null; break; } } #endregion } else { // Unable to fetch one of the indexes. Stop Interestion !! resultCacheIndexInternal = null; indexIdIndexHeaderMapping = null; break; } #region Get MetaData if (intersectionQuery.GetIndexHeader) { if (!indexIdIndexHeaderMapping.ContainsKey(indexId)) { indexIdIndexHeaderMapping.Add(indexId, GetIndexHeader(targetIndex, indexTypeMapping, messageContext.TypeId, IndexCacheUtils.GeneratePrimaryId(indexId), storeContext)); } } #endregion } if (resultCacheIndexInternal != null && resultCacheIndexInternal.Count > 0) { resultItemList = CacheIndexInternalAdapter.GetIndexDataItemList(resultCacheIndexInternal, 1, int.MaxValue); } #endregion #region Get data if (!intersectionQuery.ExcludeData && resultItemList != null && resultItemList.Count > 0) { DataTierUtil.GetData(resultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, intersectionQuery.FullDataIdInfo); } #endregion } intersectionQueryResult = new IntersectionQueryResult(resultItemList, indexIdIndexHeaderMapping, localIdentityTagNames, isTagPrimarySort, sortFieldName, sortOrderList, exceptionInfo.ToString()); // update performance counter PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.IndexLookupAvgPerIntersectionQuery, messageContext.TypeId, intersectionQuery.IndexIdList.Count); } catch (Exception ex) { exceptionInfo.Append(" | " + ex.Message); intersectionQueryResult = new IntersectionQueryResult(null, null, null, false, null, null, exceptionInfo.ToString()); LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing IntersectionQuery : {1}", messageContext.TypeId, ex); } return intersectionQueryResult; }
/// <summary> /// Processes the paged index query. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessPagedIndexQuery(RelayMessage message, MessageContext messageContext) { PagedIndexQueryResult pagedIndexQueryResult = PagedQueryProcessor.Instance.Process(message.GetQueryObject<PagedIndexQuery>(), messageContext, storeContext); bool compressOption = storeContext.GetCompressOption(message.TypeId); message.Payload = new RelayPayload(message.TypeId, message.Id, Serializer.Serialize(pagedIndexQueryResult, compressOption), compressOption); }
/// <summary> /// Processes the save message. /// </summary> /// <param name="message">The message.</param> /// <param name="messageContext">The message context.</param> private void ProcessSaveMessage(RelayMessage message, MessageContext messageContext) { SaveProcessor.Process(message.GetObject<CacheIndex>(), messageContext, storeContext); }
/// <summary> /// Process a RemoteClusteredSpanQuery /// </summary> /// <param name="message"></param> /// <param name="messageContext"></param> private void ProcessRemoteClusteredSpanQuery(RelayMessage message, MessageContext messageContext) { SpanQueryResult spanQueryResult = RemoteClusteredSpanQueryProcessor.Instance.Process( message.GetQueryObject<RemoteClusteredSpanQuery>(), messageContext, storeContext); bool compressOption = storeContext.GetCompressOption(message.TypeId); message.Payload = new RelayPayload(message.TypeId, message.Id, Serializer.Serialize(spanQueryResult, compressOption), compressOption); }
/// <summary> /// Processes the specified first last query. /// </summary> /// <param name="firstLastQuery">The first last query.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> /// <returns>FirstLastQueryResult</returns> internal static FirstLastQueryResult Process(FirstLastQuery firstLastQuery, MessageContext messageContext, IndexStoreContext storeContext) { FirstLastQueryResult firstLastQueryResult; List<ResultItem> firstPageResultItemList = null; List<ResultItem> lastPageResultItemList = null; bool indexExists = false; int indexSize = -1; int virtualCount = -1; byte[] metadata = null; try { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; #region Validate Query ValidateQuery(indexTypeMapping, firstLastQuery); #endregion int maxItemsPerIndex = ((firstLastQuery.FirstPageSize > 0) && (firstLastQuery.LastPageSize < 1)) ? firstLastQuery.FirstPageSize : int.MaxValue; Index targetIndexInfo = indexTypeMapping.IndexCollection[firstLastQuery.TargetIndexName]; #region Prepare Result #region Extract index and apply criteria CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, firstLastQuery.IndexId, targetIndexInfo.ExtendedIdSuffix, firstLastQuery.TargetIndexName, maxItemsPerIndex, firstLastQuery.Filter, true, null, false, false, targetIndexInfo.PrimarySortInfo, targetIndexInfo.LocalIdentityTagList, targetIndexInfo.StringHashCodeDictionary, null); #endregion if (targetIndex != null) { indexSize = targetIndex.OutDeserializationContext.TotalCount; virtualCount = targetIndex.VirtualCount; indexExists = true; // update perf counters PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerFirstLastQuery, messageContext.TypeId, indexSize); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerFirstLastQuery, messageContext.TypeId, targetIndex.OutDeserializationContext.ReadItemCount); #region Populate resultLists if (firstLastQuery.FirstPageSize + firstLastQuery.LastPageSize <= targetIndex.Count) { firstPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, firstLastQuery.FirstPageSize); lastPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, targetIndex.Count - firstLastQuery.LastPageSize + 1, firstLastQuery.LastPageSize); } else { //Populate everything in firstPageResultItemList firstPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, targetIndex.Count); } #endregion #region Get data if (!firstLastQuery.ExcludeData) { //First Page if (firstPageResultItemList != null) { DataTierUtil.GetData(firstPageResultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, firstLastQuery.FullDataIdInfo); } //Last Page if (lastPageResultItemList != null) { DataTierUtil.GetData(lastPageResultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, firstLastQuery.FullDataIdInfo); } } #endregion #region Get metadata if (firstLastQuery.GetMetadata) { metadata = IndexServerUtils.GetQueryMetadata(new List<CacheIndexInternal>(1) { targetIndex }, indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, firstLastQuery.IndexId, storeContext); } #endregion } #endregion firstLastQueryResult = new FirstLastQueryResult(indexExists, indexSize, metadata, firstPageResultItemList, lastPageResultItemList, virtualCount, null); } catch (Exception ex) { firstLastQueryResult = new FirstLastQueryResult(false, -1, null, null, null, -1, ex.Message); LoggingUtil.Log.ErrorFormat("TypeID {0} -- Error processing FirstLastQuery : {1}", messageContext.TypeId, ex); } return firstLastQueryResult; }