internal static void Process(InternalItem internalItem, TagHashCollection tagHashCollection, short typeId, DomainSpecificProcessingType domainSpecificProcessingType, DomainSpecificConfig domainSpecificConfig) { switch (domainSpecificProcessingType) { case DomainSpecificProcessingType.StreamRecency: tagHashCollection.AddTag(typeId, RecencyTagName); internalItem.TagList.Add(new System.Collections.Generic.KeyValuePair <int, byte[]>(TagHashCollection.GetTagHashCode(RecencyTagName), GetRecency(internalItem, domainSpecificConfig.StreamRecencyConfig))); break; } }
internal InDeserializationContext(int maxItemsPerIndex, string indexName, byte[] indexId, short typeId, Filter filter, bool inclusiveFilter, TagHashCollection tagHashCollection, bool deserializeHeaderOnly, bool collectFilteredItems, PrimarySortInfo primarySortInfo, List <string> localIdentityTagNames, StringHashCollection stringHashCollection, Dictionary <int, bool> stringHashCodeDictionary, IndexCondition indexCondition, CapCondition capCondition, bool isMetadataPropertyCollection, MetadataPropertyCollection metadataPropertyCollection, DomainSpecificProcessingType domainSpecificProcessingType, DomainSpecificConfig domainSpecificConfig, string getDistinctValuesFieldName, GroupBy groupBy) { MaxItemsPerIndex = maxItemsPerIndex; IndexName = indexName; IndexId = indexId; TypeId = typeId; Filter = filter; InclusiveFilter = inclusiveFilter; TagHashCollection = tagHashCollection; DeserializeHeaderOnly = deserializeHeaderOnly; CollectFilteredItems = collectFilteredItems; PrimarySortInfo = primarySortInfo; LocalIdentityTagNames = localIdentityTagNames; StringHashCollection = stringHashCollection; StringHashCodeDictionary = stringHashCodeDictionary; IndexCondition = indexCondition; CapCondition = capCondition; IsMetadataPropertyCollection = isMetadataPropertyCollection; MetadataPropertyCollection = metadataPropertyCollection; DomainSpecificProcessingType = domainSpecificProcessingType; DomainSpecificConfig = domainSpecificConfig; GetDistinctValuesFieldName = getDistinctValuesFieldName; GroupBy = groupBy; SetEnterExitCondition(); }
/// <summary> /// Initializes the reload config. /// </summary> /// <param name="config">The config.</param> /// <param name="isInit">if set to <c>true</c> [is init].</param> internal void InitializeReloadConfig(RelayNodeConfig config, bool isInit) { #region Set Component Configs if (config == null) { Exception ex = new Exception("Unable to Initialize/Reload Config for CacheIndexV3Store because RelayNodeConfig is null"); LoggingUtil.Log.Error(ex.Message); throw ex; } Interlocked.Exchange(ref nodeConfig, config); Interlocked.Exchange(ref storageConfiguration, InitializeConfig(nodeConfig)); #endregion #region Setup Forwarder if (ForwarderComponent == null) { ForwarderComponent = new Forwarder(); ForwarderComponent.Initialize(config, null); } else { ForwarderComponent.ReloadConfig(config); } #endregion #region Init/Reload Component if (IndexStorageComponent == null) { IndexStorageComponent = InitializeStorageComponent(storageConfiguration.BerkeleyDbConfig); } else { ReloadStorageComponent(storageConfiguration.BerkeleyDbConfig, IndexStorageComponent); } #endregion #region init memoryPool if (this.myMemoryPool == null) { this.myMemoryPool = new MemoryStreamPool( storageConfiguration.CacheIndexV3StorageConfig.MemPoolItemInitialSizeInBytes, ResourcePool.ResourcePool <MemoryStream> .InfiniteReuse, storageConfiguration.CacheIndexV3StorageConfig.MemPoolMinItemNumber); } #region init DataMembers Interlocked.Exchange(ref relatedTypeIds, InitializeRelatedTypeIds(nodeConfig)); Interlocked.Exchange(ref compressOptions, InitializeCompressOptions(nodeConfig)); InitializeClusterInfo(out myClusterPosition, out numClustersInGroup, out myZone); LockingUtil.Instance.InitializeLockerObjects(storageConfiguration.CacheIndexV3StorageConfig.LockMultiplier, numClustersInGroup); LegacySerializationUtil.Instance.InitializeLegacySerializtionTypes(nodeConfig.TypeSettings, storageConfiguration.CacheIndexV3StorageConfig.SupportLegacySerialization); #region init performance counters List <short> typeIdList = new List <short>(); foreach (IndexTypeMapping indexTypeMapping in storageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection) { typeIdList.Add(indexTypeMapping.TypeId); } // get the max type id short maxTypeId = config.TypeSettings.MaxTypeId; // initialize or re-initialize performance counters, // counter category will also be created if it is not there PerformanceCounters.Instance.InitializeCounters( config.TransportSettings.ListenPort, typeIdList, maxTypeId, isInit); #endregion #region Set HashCollections Interlocked.Exchange(ref tagHashCollection, InitializeTagHashCollection(storageConfiguration)); Interlocked.Exchange(ref stringHashCollection, InitializeStringHashCollection(storageConfiguration)); #endregion #endregion #region Init Domain Specific Config DomainSpecificConfig = new DomainSpecificConfig { StreamRecencyConfig = ConfigurationManager.GetSection("StreamRecencyConfig") as StreamRecencyConfig }; #endregion }
/// <summary> /// Gets the CacheIndexInternal. /// </summary> /// <param name="storeContext">The store context.</param> /// <param name="typeId">The type id.</param> /// <param name="primaryId">The primary id.</param> /// <param name="indexId">The index id.</param> /// <param name="extendedIdSuffix">The extended id suffix.</param> /// <param name="indexName">Name of the index.</param> /// <param name="maxItemsPerIndex">The maxItemsPerIndex.</param> /// <param name="filter">The filter.</param> /// <param name="inclusiveFilter">if set to <c>true</c> includes the items that pass the filter; otherwise , <c>false</c>.</param> /// <param name="indexCondition">The index condition.</param> /// <param name="deserializeHeaderOnly">if set to <c>true</c> if just CacheIndexInternal header is to be deserialized; otherwise, <c>false</c>.</param> /// <param name="getFilteredItems">if set to <c>true</c> get filtered items; otherwise, <c>false</c>.</param> /// <param name="primarySortInfo">The primary sort info.</param> /// <param name="localIdentityTagNames">The local identity tag names.</param> /// <param name="stringHashCodeDictionary">The string hash code dictionary.</param> /// <param name="capCondition">The cap condition.</param> /// <param name="isMetadataPropertyCollection">if set to <c>true</c> metadata represents a property collection; otherwise , <c>false</c>.</param> /// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param> /// <param name="domainSpecificProcessingType">The DomainSpecificProcessingType.</param> /// <param name="domainSpecificConfig">The DomainSpecificConfig.</param> /// <param name="getDistinctValuesFieldName">The distinct value field name.</param> /// <param name="groupBy">The GroupBy clause.</param> /// <param name="forceFullGet">indicate whether or not use full get.</param> /// <returns>CacheIndexInternal</returns> internal static CacheIndexInternal GetCacheIndexInternal(IndexStoreContext storeContext, short typeId, int primaryId, byte[] indexId, short extendedIdSuffix, string indexName, int maxItemsPerIndex, Filter filter, bool inclusiveFilter, IndexCondition indexCondition, bool deserializeHeaderOnly, bool getFilteredItems, PrimarySortInfo primarySortInfo, List <string> localIdentityTagNames, Dictionary <int, bool> stringHashCodeDictionary, CapCondition capCondition, bool isMetadataPropertyCollection, MetadataPropertyCollection metadataPropertyCollection, DomainSpecificProcessingType domainSpecificProcessingType, DomainSpecificConfig domainSpecificConfig, string getDistinctValuesFieldName, GroupBy groupBy, bool forceFullGet) { CacheIndexInternal cacheIndexInternal = null; byte[] extendedId = FormExtendedId(indexId, extendedIdSuffix); //RelayMessage getMsg = new RelayMessage(typeId, primaryId, extendedId, MessageType.Get); //storeContext.IndexStorageComponent.HandleMessage(getMsg); Stream myStream; ResourcePoolItem <MemoryStream> pooledStreamItem = null; MemoryStream pooledStream; try { int indexLevelGetSize = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId]. IndexCollection[indexName].PartialGetSizeInBytes; bool isFullGet = forceFullGet ? true : (indexLevelGetSize <= 0 && storeContext.PartialGetLength <= 0); if (isFullGet) { byte[] cacheBytes = BinaryStorageAdapter.Get( storeContext.IndexStorageComponent, typeId, primaryId, extendedId); if (cacheBytes != null) { myStream = new MemoryStream(cacheBytes); } else { return(null); } } else { int partialGetSize = indexLevelGetSize == 0 ? storeContext.PartialGetLength : indexLevelGetSize; // get a memory stream from the memory pool pooledStreamItem = storeContext.MemoryPool.GetItem(); pooledStream = pooledStreamItem.Item; myStream = new SmartStream( storeContext.IndexStorageComponent, typeId, primaryId, extendedId, partialGetSize, pooledStream); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.PartialGetPerSec, 1); } if (myStream.Length > 0) //CacheIndex exists, this check is just for SmartStream { cacheIndexInternal = new CacheIndexInternal { InDeserializationContext = new InDeserializationContext(maxItemsPerIndex, indexName, indexId, typeId, filter, inclusiveFilter, storeContext.TagHashCollection, deserializeHeaderOnly, getFilteredItems, primarySortInfo, localIdentityTagNames, storeContext.StringHashCollection, stringHashCodeDictionary, indexCondition, capCondition, isMetadataPropertyCollection, metadataPropertyCollection, domainSpecificProcessingType, domainSpecificConfig, getDistinctValuesFieldName, groupBy) }; if (!isFullGet) { // skip the bdb entry header myStream.Read(new byte[BdbHeaderSize], 0, BdbHeaderSize); } // This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal // ** TBD - Should be removed later if (LegacySerializationUtil.Instance.IsSupported(typeId)) { cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream)); } else { int version = myStream.ReadByte(); try { cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream), version); } catch (Exception ex) { LoggingUtil.Log.ErrorFormat( "The deserialization has an exception: primary id : {0}, index id : {1}, extendedid : {2}, extendedIdSuffix : {3}, version : {4} info : {5}", primaryId, ByteArrayToString(indexId, 0), ByteArrayToString(extendedId, 0), extendedIdSuffix, version, ex.ToString()); if (myStream.Length > 0) { myStream.Seek(0, SeekOrigin.Begin); byte[] ba = new byte[10]; myStream.Read(ba, 0, 10); LoggingUtil.Log.ErrorFormat("The first 10 bytes of the stream are {0}", ByteArrayToString(ba, 0)); } throw; } } // update SmartStream perf counters if (!isFullGet) { SmartStream mySmartStream = (SmartStream)myStream; BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGet, mySmartStream.DatabaseAccessTime); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGetBase, 1); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGet, mySmartStream.Position); BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter( BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGetBase, 1); } } } finally { // release the pooled stream if (storeContext.MemoryPool != null && pooledStreamItem != null) { storeContext.MemoryPool.ReleaseItem(pooledStreamItem); } } return(cacheIndexInternal); }