Beispiel #1
0
 internal static void Process(InternalItem internalItem,
                              TagHashCollection tagHashCollection,
                              short typeId,
                              DomainSpecificProcessingType domainSpecificProcessingType,
                              DomainSpecificConfig domainSpecificConfig)
 {
     switch (domainSpecificProcessingType)
     {
     case DomainSpecificProcessingType.StreamRecency:
         tagHashCollection.AddTag(typeId, RecencyTagName);
         internalItem.TagList.Add(new System.Collections.Generic.KeyValuePair <int, byte[]>(TagHashCollection.GetTagHashCode(RecencyTagName),
                                                                                            GetRecency(internalItem, domainSpecificConfig.StreamRecencyConfig)));
         break;
     }
 }
Beispiel #2
0
        internal InDeserializationContext(int maxItemsPerIndex,
                                          string indexName,
                                          byte[] indexId,
                                          short typeId,
                                          Filter filter,
                                          bool inclusiveFilter,
                                          TagHashCollection tagHashCollection,
                                          bool deserializeHeaderOnly,
                                          bool collectFilteredItems,
                                          PrimarySortInfo primarySortInfo,
                                          List <string> localIdentityTagNames,
                                          StringHashCollection stringHashCollection,
                                          Dictionary <int, bool> stringHashCodeDictionary,
                                          IndexCondition indexCondition,
                                          CapCondition capCondition,
                                          bool isMetadataPropertyCollection,
                                          MetadataPropertyCollection metadataPropertyCollection,
                                          DomainSpecificProcessingType domainSpecificProcessingType,
                                          DomainSpecificConfig domainSpecificConfig,
                                          string getDistinctValuesFieldName,
                                          GroupBy groupBy)
        {
            MaxItemsPerIndex             = maxItemsPerIndex;
            IndexName                    = indexName;
            IndexId                      = indexId;
            TypeId                       = typeId;
            Filter                       = filter;
            InclusiveFilter              = inclusiveFilter;
            TagHashCollection            = tagHashCollection;
            DeserializeHeaderOnly        = deserializeHeaderOnly;
            CollectFilteredItems         = collectFilteredItems;
            PrimarySortInfo              = primarySortInfo;
            LocalIdentityTagNames        = localIdentityTagNames;
            StringHashCollection         = stringHashCollection;
            StringHashCodeDictionary     = stringHashCodeDictionary;
            IndexCondition               = indexCondition;
            CapCondition                 = capCondition;
            IsMetadataPropertyCollection = isMetadataPropertyCollection;
            MetadataPropertyCollection   = metadataPropertyCollection;
            DomainSpecificProcessingType = domainSpecificProcessingType;
            DomainSpecificConfig         = domainSpecificConfig;
            GetDistinctValuesFieldName   = getDistinctValuesFieldName;
            GroupBy                      = groupBy;

            SetEnterExitCondition();
        }
        /// <summary>
        /// Initializes the reload config.
        /// </summary>
        /// <param name="config">The config.</param>
        /// <param name="isInit">if set to <c>true</c> [is init].</param>
        internal void InitializeReloadConfig(RelayNodeConfig config, bool isInit)
        {
            #region Set Component Configs

            if (config == null)
            {
                Exception ex = new Exception("Unable to Initialize/Reload Config for CacheIndexV3Store because RelayNodeConfig is null");
                LoggingUtil.Log.Error(ex.Message);
                throw ex;
            }

            Interlocked.Exchange(ref nodeConfig, config);
            Interlocked.Exchange(ref storageConfiguration, InitializeConfig(nodeConfig));

            #endregion

            #region Setup Forwarder

            if (ForwarderComponent == null)
            {
                ForwarderComponent = new Forwarder();
                ForwarderComponent.Initialize(config, null);
            }
            else
            {
                ForwarderComponent.ReloadConfig(config);
            }

            #endregion

            #region Init/Reload Component

            if (IndexStorageComponent == null)
            {
                IndexStorageComponent = InitializeStorageComponent(storageConfiguration.BerkeleyDbConfig);
            }
            else
            {
                ReloadStorageComponent(storageConfiguration.BerkeleyDbConfig, IndexStorageComponent);
            }

            #endregion

            #region init memoryPool
            if (this.myMemoryPool == null)
            {
                this.myMemoryPool = new MemoryStreamPool(
                    storageConfiguration.CacheIndexV3StorageConfig.MemPoolItemInitialSizeInBytes,
                    ResourcePool.ResourcePool <MemoryStream> .InfiniteReuse,
                    storageConfiguration.CacheIndexV3StorageConfig.MemPoolMinItemNumber);
            }

            #region init DataMembers

            Interlocked.Exchange(ref relatedTypeIds, InitializeRelatedTypeIds(nodeConfig));

            Interlocked.Exchange(ref compressOptions, InitializeCompressOptions(nodeConfig));

            InitializeClusterInfo(out myClusterPosition, out numClustersInGroup, out myZone);

            LockingUtil.Instance.InitializeLockerObjects(storageConfiguration.CacheIndexV3StorageConfig.LockMultiplier, numClustersInGroup);

            LegacySerializationUtil.Instance.InitializeLegacySerializtionTypes(nodeConfig.TypeSettings, storageConfiguration.CacheIndexV3StorageConfig.SupportLegacySerialization);


            #region init performance counters

            List <short> typeIdList = new List <short>();
            foreach (IndexTypeMapping indexTypeMapping in storageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection)
            {
                typeIdList.Add(indexTypeMapping.TypeId);
            }

            // get the max type id
            short maxTypeId = config.TypeSettings.MaxTypeId;

            // initialize or re-initialize performance counters,
            // counter category will also be created if it is not there
            PerformanceCounters.Instance.InitializeCounters(
                config.TransportSettings.ListenPort,
                typeIdList,
                maxTypeId,
                isInit);

            #endregion

            #region Set HashCollections

            Interlocked.Exchange(ref tagHashCollection, InitializeTagHashCollection(storageConfiguration));
            Interlocked.Exchange(ref stringHashCollection, InitializeStringHashCollection(storageConfiguration));

            #endregion

            #endregion

            #region Init Domain Specific Config

            DomainSpecificConfig = new DomainSpecificConfig
            {
                StreamRecencyConfig = ConfigurationManager.GetSection("StreamRecencyConfig") as StreamRecencyConfig
            };

            #endregion
        }
Beispiel #4
0
        /// <summary>
        /// Gets the CacheIndexInternal.
        /// </summary>
        /// <param name="storeContext">The store context.</param>
        /// <param name="typeId">The type id.</param>
        /// <param name="primaryId">The primary id.</param>
        /// <param name="indexId">The index id.</param>
        /// <param name="extendedIdSuffix">The extended id suffix.</param>
        /// <param name="indexName">Name of the index.</param>
        /// <param name="maxItemsPerIndex">The maxItemsPerIndex.</param>
        /// <param name="filter">The filter.</param>
        /// <param name="inclusiveFilter">if set to <c>true</c> includes the items that pass the filter; otherwise , <c>false</c>.</param>
        /// <param name="indexCondition">The index condition.</param>
        /// <param name="deserializeHeaderOnly">if set to <c>true</c> if just CacheIndexInternal header is to be deserialized; otherwise, <c>false</c>.</param>
        /// <param name="getFilteredItems">if set to <c>true</c> get filtered items; otherwise, <c>false</c>.</param>
        /// <param name="primarySortInfo">The primary sort info.</param>
        /// <param name="localIdentityTagNames">The local identity tag names.</param>
        /// <param name="stringHashCodeDictionary">The string hash code dictionary.</param>
        /// <param name="capCondition">The cap condition.</param>
        /// <param name="isMetadataPropertyCollection">if set to <c>true</c> metadata represents a property collection; otherwise , <c>false</c>.</param>
        /// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param>
        /// <param name="domainSpecificProcessingType">The DomainSpecificProcessingType.</param>
        /// <param name="domainSpecificConfig">The DomainSpecificConfig.</param>
        /// <param name="getDistinctValuesFieldName">The distinct value field name.</param>
        /// <param name="groupBy">The GroupBy clause.</param>
        /// <param name="forceFullGet">indicate whether or not use full get.</param>
        /// <returns>CacheIndexInternal</returns>
        internal static CacheIndexInternal GetCacheIndexInternal(IndexStoreContext storeContext,
                                                                 short typeId,
                                                                 int primaryId,
                                                                 byte[] indexId,
                                                                 short extendedIdSuffix,
                                                                 string indexName,
                                                                 int maxItemsPerIndex,
                                                                 Filter filter,
                                                                 bool inclusiveFilter,
                                                                 IndexCondition indexCondition,
                                                                 bool deserializeHeaderOnly,
                                                                 bool getFilteredItems,
                                                                 PrimarySortInfo primarySortInfo,
                                                                 List <string> localIdentityTagNames,
                                                                 Dictionary <int, bool> stringHashCodeDictionary,
                                                                 CapCondition capCondition,
                                                                 bool isMetadataPropertyCollection,
                                                                 MetadataPropertyCollection metadataPropertyCollection,
                                                                 DomainSpecificProcessingType domainSpecificProcessingType,
                                                                 DomainSpecificConfig domainSpecificConfig,
                                                                 string getDistinctValuesFieldName,
                                                                 GroupBy groupBy,
                                                                 bool forceFullGet)
        {
            CacheIndexInternal cacheIndexInternal = null;

            byte[] extendedId = FormExtendedId(indexId, extendedIdSuffix);
            //RelayMessage getMsg = new RelayMessage(typeId, primaryId, extendedId, MessageType.Get);
            //storeContext.IndexStorageComponent.HandleMessage(getMsg);

            Stream myStream;

            ResourcePoolItem <MemoryStream> pooledStreamItem = null;
            MemoryStream pooledStream;

            try
            {
                int indexLevelGetSize =
                    storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId].
                    IndexCollection[indexName].PartialGetSizeInBytes;

                bool isFullGet = forceFullGet ? true : (indexLevelGetSize <= 0 && storeContext.PartialGetLength <= 0);

                if (isFullGet)
                {
                    byte[] cacheBytes = BinaryStorageAdapter.Get(
                        storeContext.IndexStorageComponent,
                        typeId,
                        primaryId,
                        extendedId);

                    if (cacheBytes != null)
                    {
                        myStream = new MemoryStream(cacheBytes);
                    }
                    else
                    {
                        return(null);
                    }
                }
                else
                {
                    int partialGetSize = indexLevelGetSize == 0 ? storeContext.PartialGetLength : indexLevelGetSize;

                    // get a memory stream from the memory pool
                    pooledStreamItem = storeContext.MemoryPool.GetItem();
                    pooledStream     = pooledStreamItem.Item;

                    myStream = new SmartStream(
                        storeContext.IndexStorageComponent,
                        typeId,
                        primaryId,
                        extendedId,
                        partialGetSize,
                        pooledStream);

                    BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
                        BerkeleyBinaryStorePerformanceCounterEnum.PartialGetPerSec,
                        1);
                }

                if (myStream.Length > 0) //CacheIndex exists, this check is just for SmartStream
                {
                    cacheIndexInternal = new CacheIndexInternal
                    {
                        InDeserializationContext =
                            new InDeserializationContext(maxItemsPerIndex,
                                                         indexName,
                                                         indexId,
                                                         typeId,
                                                         filter,
                                                         inclusiveFilter,
                                                         storeContext.TagHashCollection,
                                                         deserializeHeaderOnly,
                                                         getFilteredItems,
                                                         primarySortInfo,
                                                         localIdentityTagNames,
                                                         storeContext.StringHashCollection,
                                                         stringHashCodeDictionary,
                                                         indexCondition,
                                                         capCondition,
                                                         isMetadataPropertyCollection,
                                                         metadataPropertyCollection,
                                                         domainSpecificProcessingType,
                                                         domainSpecificConfig,
                                                         getDistinctValuesFieldName,
                                                         groupBy)
                    };

                    if (!isFullGet)
                    {
                        // skip the bdb entry header
                        myStream.Read(new byte[BdbHeaderSize], 0, BdbHeaderSize);
                    }

                    // This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal
                    // ** TBD - Should be removed later
                    if (LegacySerializationUtil.Instance.IsSupported(typeId))
                    {
                        cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream));
                    }
                    else
                    {
                        int version = myStream.ReadByte();

                        try
                        {
                            cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream), version);
                        }
                        catch (Exception ex)
                        {
                            LoggingUtil.Log.ErrorFormat(
                                "The deserialization has an exception: primary id : {0}, index id : {1}, extendedid : {2}, extendedIdSuffix : {3}, version : {4} info : {5}",
                                primaryId,
                                ByteArrayToString(indexId, 0),
                                ByteArrayToString(extendedId, 0),
                                extendedIdSuffix,
                                version,
                                ex.ToString());

                            if (myStream.Length > 0)
                            {
                                myStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[10];
                                myStream.Read(ba, 0, 10);

                                LoggingUtil.Log.ErrorFormat("The first 10 bytes of the stream are {0}",
                                                            ByteArrayToString(ba, 0));
                            }

                            throw;
                        }
                    }

                    // update SmartStream perf counters
                    if (!isFullGet)
                    {
                        SmartStream mySmartStream = (SmartStream)myStream;

                        BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
                            BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGet,
                            mySmartStream.DatabaseAccessTime);

                        BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
                            BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGetBase,
                            1);

                        BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
                            BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGet,
                            mySmartStream.Position);

                        BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
                            BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGetBase,
                            1);
                    }
                }
            }
            finally
            {
                // release the pooled stream
                if (storeContext.MemoryPool != null && pooledStreamItem != null)
                {
                    storeContext.MemoryPool.ReleaseItem(pooledStreamItem);
                }
            }

            return(cacheIndexInternal);
        }