/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MessageContext messageContext, IndexStoreContext storeContext) { // TBD : Support concurrent DeleteAllInType for different types lock (LockingUtil.Instance.LockerObjects) { RelayMessage msg; short typeId = messageContext.TypeId; if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId].IndexServerMode)) { // Send DeleteAll to Data Store short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(typeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", typeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + typeId); } msg = new RelayMessage(relatedTypeId, 0, MessageType.DeleteAllInType); storeContext.ForwarderComponent.HandleMessage(msg); } // Send DeleteAll to local Index storage BinaryStorageAdapter.Clear(storeContext.IndexStorageComponent, typeId); // Delete hash mapping entries in file storeContext.RemoveType(typeId); } }
/// <summary> /// Gets the metadata when its stored seperately. /// </summary> /// <param name="indexTypeMapping">The index type mapping.</param> /// <param name="typeId">The type id.</param> /// <param name="primaryId">The primary id.</param> /// <param name="extendedId">The extended id.</param> /// <param name="storeContext">The store context.</param> /// <param name="metadata">The Metadata.</param> /// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param> internal static void GetMetadataStoredSeperately(IndexTypeMapping indexTypeMapping, short typeId, int primaryId, byte[] extendedId, IndexStoreContext storeContext, out byte[] metadata, out MetadataPropertyCollection metadataPropertyCollection) { metadata = null; metadataPropertyCollection = null; byte[] metadataBytes = BinaryStorageAdapter.Get(storeContext.IndexStorageComponent, typeId, primaryId, extendedId); if (metadataBytes != null) { if (indexTypeMapping.IsMetadataPropertyCollection) { metadataPropertyCollection = GetMetadataPropertyCollection(metadataBytes); } else { metadata = metadataBytes; } } }
/// <summary> /// Processes the specified filtered index delete command. /// </summary> /// <param name="filteredIndexDeleteCommand">The filtered index delete command.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process( FilteredIndexDeleteCommand filteredIndexDeleteCommand, MessageContext messageContext, IndexStoreContext storeContext) { if (filteredIndexDeleteCommand != null) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; Index indexInfo = indexTypeMapping.IndexCollection[filteredIndexDeleteCommand.TargetIndexName]; byte[] metadata = null; MetadataPropertyCollection metadataPropertyCollection = null; #region Get CacheIndexInternal for TargetIndexName if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, messageContext.PrimaryId, filteredIndexDeleteCommand.IndexId, storeContext, out metadata, out metadataPropertyCollection); } CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, filteredIndexDeleteCommand.IndexId, indexInfo.ExtendedIdSuffix, filteredIndexDeleteCommand.TargetIndexName, 0, filteredIndexDeleteCommand.DeleteFilter, false, null, false, true, indexInfo.PrimarySortInfo, indexInfo.LocalIdentityTagList, indexInfo.StringHashCodeDictionary, null, indexInfo.IsMetadataPropertyCollection, metadataPropertyCollection, DomainSpecificProcessingType.None, null, null, null, false); #endregion if (cacheIndexInternal != null) { #region Increment perf counters' number PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsInIndexPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.TotalCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsReadPerFilterDeleteRequest, messageContext.TypeId, cacheIndexInternal.OutDeserializationContext.ReadItemCount); PerformanceCounters.Instance.SetCounterValue( PerformanceCounterEnum.NumOfItemsFilteredPerFilterDeleteRequest, messageContext.TypeId, (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count)); #endregion #region Update VirtualCount cacheIndexInternal.VirtualCount -= (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.Count); #endregion #region Save CacheIndexInternal to local storage since item which pass delete filter are pruned in it byte[] extendedId = IndexServerUtils.FormExtendedId(filteredIndexDeleteCommand.IndexId, indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix); // compose a bdb entry header bool isCompress = storeContext.GetCompressOption(messageContext.TypeId); PayloadStorage bdbEntryHeader = new PayloadStorage { Compressed = isCompress, TTL = -1, LastUpdatedTicks = DateTime.Now.Ticks, ExpirationTicks = -1, Deactivated = false }; BinaryStorageAdapter.Save( storeContext.MemoryPool, storeContext.IndexStorageComponent, messageContext.TypeId, filteredIndexDeleteCommand.PrimaryId, extendedId, bdbEntryHeader, Serializer.Serialize <CacheIndexInternal>(cacheIndexInternal, isCompress, RelayMessage.RelayCompressionImplementation)); #endregion #region Data store deletes if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode) && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList != null && cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count > 0) { List <RelayMessage> dataStorageMessageList = new List <RelayMessage>(); short relatedTypeId; if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } cacheIndexInternal.InternalItemList = cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList; List <byte[]> fullDataIdList = DataTierUtil.GetFullDataIds(cacheIndexInternal.InDeserializationContext.IndexId, cacheIndexInternal.InternalItemList, indexTypeMapping.FullDataIdFieldList); foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } } #endregion } } }
/// <summary> /// Processes the specified message context. /// </summary> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MessageContext messageContext, IndexStoreContext storeContext) { lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId)) { IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; List <RelayMessage> indexStorageMessageList = new List <RelayMessage>(indexTypeMapping.IndexCollection.Count); List <RelayMessage> dataStorageMessageList = new List <RelayMessage>(); CacheIndexInternal internalIndex; List <byte[]> fullDataIdList; if (indexTypeMapping.MetadataStoredSeperately) { BinaryStorageAdapter.Delete( storeContext.IndexStorageComponent, messageContext.TypeId, messageContext.PrimaryId, messageContext.ExtendedId); } foreach (Index index in indexTypeMapping.IndexCollection) { internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, messageContext.PrimaryId, messageContext.ExtendedId, index.ExtendedIdSuffix, index.IndexName, 0, null, true, null, false, false, index.PrimarySortInfo, index.LocalIdentityTagList, index.StringHashCodeDictionary, null, indexTypeMapping.IndexCollection[index.IndexName].IsMetadataPropertyCollection, null, DomainSpecificProcessingType.None, null, null, null, true); if (internalIndex != null) { #region Deletes messages for data store if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL, messageContext.SourceZone, storeContext.MyZone, indexTypeMapping.IndexServerMode)) { fullDataIdList = DataTierUtil.GetFullDataIds(messageContext.ExtendedId, internalIndex.InternalItemList, indexTypeMapping.FullDataIdFieldList); short relatedTypeId; foreach (byte[] fullDataId in fullDataIdList) { if (fullDataId != null) { if (storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId)) { dataStorageMessageList.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(fullDataId), fullDataId, MessageType.Delete)); } else { LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId); throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId); } } } } #endregion #region Delete messages for index store BinaryStorageAdapter.Delete( storeContext.IndexStorageComponent, messageContext.TypeId, messageContext.PrimaryId, IndexServerUtils.FormExtendedId(messageContext.ExtendedId, index.ExtendedIdSuffix)); #endregion } } if (dataStorageMessageList.Count > 0) { storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList); } } }
/// <summary> /// Gets the CacheIndexInternal. /// </summary> /// <param name="storeContext">The store context.</param> /// <param name="typeId">The type id.</param> /// <param name="primaryId">The primary id.</param> /// <param name="indexId">The index id.</param> /// <param name="extendedIdSuffix">The extended id suffix.</param> /// <param name="indexName">Name of the index.</param> /// <param name="maxItemsPerIndex">The maxItemsPerIndex.</param> /// <param name="filter">The filter.</param> /// <param name="inclusiveFilter">if set to <c>true</c> includes the items that pass the filter; otherwise , <c>false</c>.</param> /// <param name="indexCondition">The index condition.</param> /// <param name="deserializeHeaderOnly">if set to <c>true</c> if just CacheIndexInternal header is to be deserialized; otherwise, <c>false</c>.</param> /// <param name="getFilteredItems">if set to <c>true</c> get filtered items; otherwise, <c>false</c>.</param> /// <param name="primarySortInfo">The primary sort info.</param> /// <param name="localIdentityTagNames">The local identity tag names.</param> /// <param name="stringHashCodeDictionary">The string hash code dictionary.</param> /// <param name="capCondition">The cap condition.</param> /// <param name="isMetadataPropertyCollection">if set to <c>true</c> metadata represents a property collection; otherwise , <c>false</c>.</param> /// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param> /// <param name="domainSpecificProcessingType">The DomainSpecificProcessingType.</param> /// <param name="domainSpecificConfig">The DomainSpecificConfig.</param> /// <param name="getDistinctValuesFieldName">The distinct value field name.</param> /// <param name="groupBy">The GroupBy clause.</param> /// <param name="forceFullGet">indicate whether or not use full get.</param> /// <returns>CacheIndexInternal</returns> internal static CacheIndexInternal GetCacheIndexInternal(IndexStoreContext storeContext, short typeId, int primaryId, byte[] indexId, short extendedIdSuffix, string indexName, int maxItemsPerIndex, Filter filter, bool inclusiveFilter, IndexCondition indexCondition, bool deserializeHeaderOnly, bool getFilteredItems, PrimarySortInfo primarySortInfo, List <string> localIdentityTagNames, Dictionary <int, bool> stringHashCodeDictionary, CapCondition capCondition, bool isMetadataPropertyCollection, MetadataPropertyCollection metadataPropertyCollection, DomainSpecificProcessingType domainSpecificProcessingType, DomainSpecificConfig domainSpecificConfig, string getDistinctValuesFieldName, GroupBy groupBy, bool forceFullGet) { CacheIndexInternal cacheIndexInternal = null; byte[] extendedId = FormExtendedId(indexId, extendedIdSuffix); //RelayMessage getMsg = new RelayMessage(typeId, primaryId, extendedId, MessageType.Get); //storeContext.IndexStorageComponent.HandleMessage(getMsg); Stream myStream; ResourcePoolItem <MemoryStream> pooledStreamItem = null; MemoryStream pooledStream; try { int indexLevelGetSize = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId]. IndexCollection[indexName].PartialGetSizeInBytes; bool isFullGet = forceFullGet ? true : (indexLevelGetSize <= 0 && storeContext.PartialGetLength <= 0); if (isFullGet) { byte[] cacheBytes = BinaryStorageAdapter.Get( storeContext.IndexStorageComponent, typeId, primaryId, extendedId); if (cacheBytes != null) { myStream = new MemoryStream(cacheBytes); } else { return(null); } } else { int partialGetSize = indexLevelGetSize == 0 ? storeContext.PartialGetLength : indexLevelGetSize; // get a memory stream from the memory pool pooledStreamItem = storeContext.MemoryPool.GetItem(); pooledStream = pooledStreamItem.Item; myStream = new SmartStream( storeContext.IndexStorageComponent, typeId, primaryId, extendedId, partialGetSize, pooledStream); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.PartialGetPerSec, 1); } if (myStream.Length > 0) //CacheIndex exists, this check is just for SmartStream { cacheIndexInternal = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext(maxItemsPerIndex, indexName, indexId, typeId, filter, inclusiveFilter, storeContext.TagHashCollection, deserializeHeaderOnly, getFilteredItems, primarySortInfo, localIdentityTagNames, storeContext.StringHashCollection, stringHashCodeDictionary, indexCondition, capCondition, isMetadataPropertyCollection, metadataPropertyCollection, domainSpecificProcessingType, domainSpecificConfig, getDistinctValuesFieldName, groupBy) }; if (!isFullGet) { // skip the bdb entry header myStream.Read(new byte[BdbHeaderSize], 0, BdbHeaderSize); } // This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal // ** TBD - Should be removed later if (LegacySerializationUtil.Instance.IsSupported(typeId)) { cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream)); } else { int version = myStream.ReadByte(); try { cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream), version); } catch (Exception ex) { LoggingUtil.Log.ErrorFormat( "The deserialization has an exception: primary id : {0}, index id : {1}, extendedid : {2}, extendedIdSuffix : {3}, version : {4} info : {5}", primaryId, ByteArrayToString(indexId, 0), ByteArrayToString(extendedId, 0), extendedIdSuffix, version, ex.ToString()); if (myStream.Length > 0) { myStream.Seek(0, SeekOrigin.Begin); byte[] ba = new byte[10]; myStream.Read(ba, 0, 10); LoggingUtil.Log.ErrorFormat("The first 10 bytes of the stream are {0}", ByteArrayToString(ba, 0)); } throw; } } // update SmartStream perf counters if (!isFullGet) { SmartStream mySmartStream = (SmartStream)myStream; BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGet, mySmartStream.DatabaseAccessTime); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGetBase, 1); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGet, mySmartStream.Position); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGetBase, 1); } } } finally { // release the pooled stream if (storeContext.MemoryPool != null && pooledStreamItem != null) { storeContext.MemoryPool.ReleaseItem(pooledStreamItem); } } return(cacheIndexInternal); }
/// <summary> /// Processes the specified MetadataPropertyCommand. /// </summary> /// <param name="metadataPropertyCommand">The MetadataPropertyCommand.</param> /// <param name="messageContext">The message context.</param> /// <param name="storeContext">The store context.</param> internal static void Process(MetadataPropertyCommand metadataPropertyCommand, MessageContext messageContext, IndexStoreContext storeContext) { if (metadataPropertyCommand != null) { MetadataPropertyCollection metadataPropertyCollection = null; byte[] metadata; IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId]; CacheIndexInternal cacheIndexInternal = null; #region Fetch MetadataPropertyCollection if (indexTypeMapping.MetadataStoredSeperately) { IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping, messageContext.TypeId, metadataPropertyCommand.PrimaryId, metadataPropertyCommand.IndexId, storeContext, out metadata, out metadataPropertyCollection); } else { Index indexInfo = indexTypeMapping.IndexCollection[metadataPropertyCommand.TargetIndexName]; // Get CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext, messageContext.TypeId, metadataPropertyCommand.PrimaryId, metadataPropertyCommand.IndexId, indexInfo.ExtendedIdSuffix, metadataPropertyCommand.TargetIndexName, 0, null, false, null, true, false, null, null, null, null, true, null, DomainSpecificProcessingType.None, null, null, null, true); if (cacheIndexInternal != null && cacheIndexInternal.MetadataPropertyCollection != null) { metadataPropertyCollection = cacheIndexInternal.MetadataPropertyCollection; } } #endregion #region Process MetadataPropertyCollection add/deletes if (metadataPropertyCollection == null) { metadataPropertyCollection = new MetadataPropertyCollection(); } metadataPropertyCollection.Process(metadataPropertyCommand.MetadataPropertyCollectionUpdate); #endregion #region Restore MetadataPropertyCollection back to storage bool isCompress = storeContext.GetCompressOption(messageContext.TypeId); byte[] extId = null; byte[] byteArray = null; if (indexTypeMapping.MetadataStoredSeperately) { extId = metadataPropertyCommand.ExtendedId; byteArray = Serializer.Serialize(metadataPropertyCollection, isCompress); } else { if (cacheIndexInternal == null) { cacheIndexInternal = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext(0, metadataPropertyCommand.TargetIndexName, metadataPropertyCommand.IndexId, messageContext.TypeId, null, false, null, true, false, null, null, null, null, null, null, true, null, DomainSpecificProcessingType.None, null, null, null) }; } //Restore CacheIndexInternal cacheIndexInternal.MetadataPropertyCollection = metadataPropertyCollection; extId = IndexServerUtils.FormExtendedId(metadataPropertyCommand.IndexId, indexTypeMapping.IndexCollection[ cacheIndexInternal.InDeserializationContext.IndexName]. ExtendedIdSuffix); byteArray = Serializer.Serialize(cacheIndexInternal, isCompress); } PayloadStorage bdbEntryHeader = new PayloadStorage { Compressed = isCompress, TTL = -1, LastUpdatedTicks = DateTime.Now.Ticks, ExpirationTicks = -1, Deactivated = false }; BinaryStorageAdapter.Save( storeContext.MemoryPool, storeContext.IndexStorageComponent, messageContext.TypeId, metadataPropertyCommand.PrimaryId, extId, bdbEntryHeader, byteArray); #endregion } }