internal static void Process(InternalItem internalItem,
TagHashCollection tagHashCollection,
short typeId,
DomainSpecificProcessingType domainSpecificProcessingType,
DomainSpecificConfig domainSpecificConfig)
{
switch (domainSpecificProcessingType)
{
case DomainSpecificProcessingType.StreamRecency:
tagHashCollection.AddTag(typeId, RecencyTagName);
internalItem.TagList.Add(new System.Collections.Generic.KeyValuePair <int, byte[]>(TagHashCollection.GetTagHashCode(RecencyTagName),
GetRecency(internalItem, domainSpecificConfig.StreamRecencyConfig)));
break;
}
}
internal InDeserializationContext(int maxItemsPerIndex,
string indexName,
byte[] indexId,
short typeId,
Filter filter,
bool inclusiveFilter,
TagHashCollection tagHashCollection,
bool deserializeHeaderOnly,
bool collectFilteredItems,
PrimarySortInfo primarySortInfo,
List <string> localIdentityTagNames,
StringHashCollection stringHashCollection,
Dictionary <int, bool> stringHashCodeDictionary,
IndexCondition indexCondition,
CapCondition capCondition,
bool isMetadataPropertyCollection,
MetadataPropertyCollection metadataPropertyCollection,
DomainSpecificProcessingType domainSpecificProcessingType,
DomainSpecificConfig domainSpecificConfig,
string getDistinctValuesFieldName,
GroupBy groupBy)
{
MaxItemsPerIndex = maxItemsPerIndex;
IndexName = indexName;
IndexId = indexId;
TypeId = typeId;
Filter = filter;
InclusiveFilter = inclusiveFilter;
TagHashCollection = tagHashCollection;
DeserializeHeaderOnly = deserializeHeaderOnly;
CollectFilteredItems = collectFilteredItems;
PrimarySortInfo = primarySortInfo;
LocalIdentityTagNames = localIdentityTagNames;
StringHashCollection = stringHashCollection;
StringHashCodeDictionary = stringHashCodeDictionary;
IndexCondition = indexCondition;
CapCondition = capCondition;
IsMetadataPropertyCollection = isMetadataPropertyCollection;
MetadataPropertyCollection = metadataPropertyCollection;
DomainSpecificProcessingType = domainSpecificProcessingType;
DomainSpecificConfig = domainSpecificConfig;
GetDistinctValuesFieldName = getDistinctValuesFieldName;
GroupBy = groupBy;
SetEnterExitCondition();
}
/// <summary>
/// Initializes the reload config.
/// </summary>
/// <param name="config">The config.</param>
/// <param name="isInit">if set to <c>true</c> [is init].</param>
internal void InitializeReloadConfig(RelayNodeConfig config, bool isInit)
{
#region Set Component Configs
if (config == null)
{
Exception ex = new Exception("Unable to Initialize/Reload Config for CacheIndexV3Store because RelayNodeConfig is null");
LoggingUtil.Log.Error(ex.Message);
throw ex;
}
Interlocked.Exchange(ref nodeConfig, config);
Interlocked.Exchange(ref storageConfiguration, InitializeConfig(nodeConfig));
#endregion
#region Setup Forwarder
if (ForwarderComponent == null)
{
ForwarderComponent = new Forwarder();
ForwarderComponent.Initialize(config, null);
}
else
{
ForwarderComponent.ReloadConfig(config);
}
#endregion
#region Init/Reload Component
if (IndexStorageComponent == null)
{
IndexStorageComponent = InitializeStorageComponent(storageConfiguration.BerkeleyDbConfig);
}
else
{
ReloadStorageComponent(storageConfiguration.BerkeleyDbConfig, IndexStorageComponent);
}
#endregion
#region init memoryPool
if (this.myMemoryPool == null)
{
this.myMemoryPool = new MemoryStreamPool(
storageConfiguration.CacheIndexV3StorageConfig.MemPoolItemInitialSizeInBytes,
ResourcePool.ResourcePool <MemoryStream> .InfiniteReuse,
storageConfiguration.CacheIndexV3StorageConfig.MemPoolMinItemNumber);
}
#region init DataMembers
Interlocked.Exchange(ref relatedTypeIds, InitializeRelatedTypeIds(nodeConfig));
Interlocked.Exchange(ref compressOptions, InitializeCompressOptions(nodeConfig));
InitializeClusterInfo(out myClusterPosition, out numClustersInGroup, out myZone);
LockingUtil.Instance.InitializeLockerObjects(storageConfiguration.CacheIndexV3StorageConfig.LockMultiplier, numClustersInGroup);
LegacySerializationUtil.Instance.InitializeLegacySerializtionTypes(nodeConfig.TypeSettings, storageConfiguration.CacheIndexV3StorageConfig.SupportLegacySerialization);
#region init performance counters
List <short> typeIdList = new List <short>();
foreach (IndexTypeMapping indexTypeMapping in storageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection)
{
typeIdList.Add(indexTypeMapping.TypeId);
}
// get the max type id
short maxTypeId = config.TypeSettings.MaxTypeId;
// initialize or re-initialize performance counters,
// counter category will also be created if it is not there
PerformanceCounters.Instance.InitializeCounters(
config.TransportSettings.ListenPort,
typeIdList,
maxTypeId,
isInit);
#endregion
#region Set HashCollections
Interlocked.Exchange(ref tagHashCollection, InitializeTagHashCollection(storageConfiguration));
Interlocked.Exchange(ref stringHashCollection, InitializeStringHashCollection(storageConfiguration));
#endregion
#endregion
#region Init Domain Specific Config
DomainSpecificConfig = new DomainSpecificConfig
{
StreamRecencyConfig = ConfigurationManager.GetSection("StreamRecencyConfig") as StreamRecencyConfig
};
#endregion
}
/// <summary>
/// Gets the CacheIndexInternal.
/// </summary>
/// <param name="storeContext">The store context.</param>
/// <param name="typeId">The type id.</param>
/// <param name="primaryId">The primary id.</param>
/// <param name="indexId">The index id.</param>
/// <param name="extendedIdSuffix">The extended id suffix.</param>
/// <param name="indexName">Name of the index.</param>
/// <param name="maxItemsPerIndex">The maxItemsPerIndex.</param>
/// <param name="filter">The filter.</param>
/// <param name="inclusiveFilter">if set to <c>true</c> includes the items that pass the filter; otherwise , <c>false</c>.</param>
/// <param name="indexCondition">The index condition.</param>
/// <param name="deserializeHeaderOnly">if set to <c>true</c> if just CacheIndexInternal header is to be deserialized; otherwise, <c>false</c>.</param>
/// <param name="getFilteredItems">if set to <c>true</c> get filtered items; otherwise, <c>false</c>.</param>
/// <param name="primarySortInfo">The primary sort info.</param>
/// <param name="localIdentityTagNames">The local identity tag names.</param>
/// <param name="stringHashCodeDictionary">The string hash code dictionary.</param>
/// <param name="capCondition">The cap condition.</param>
/// <param name="isMetadataPropertyCollection">if set to <c>true</c> metadata represents a property collection; otherwise , <c>false</c>.</param>
/// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param>
/// <param name="domainSpecificProcessingType">The DomainSpecificProcessingType.</param>
/// <param name="domainSpecificConfig">The DomainSpecificConfig.</param>
/// <param name="getDistinctValuesFieldName">The distinct value field name.</param>
/// <param name="groupBy">The GroupBy clause.</param>
/// <param name="forceFullGet">indicate whether or not use full get.</param>
/// <returns>CacheIndexInternal</returns>
internal static CacheIndexInternal GetCacheIndexInternal(IndexStoreContext storeContext,
short typeId,
int primaryId,
byte[] indexId,
short extendedIdSuffix,
string indexName,
int maxItemsPerIndex,
Filter filter,
bool inclusiveFilter,
IndexCondition indexCondition,
bool deserializeHeaderOnly,
bool getFilteredItems,
PrimarySortInfo primarySortInfo,
List <string> localIdentityTagNames,
Dictionary <int, bool> stringHashCodeDictionary,
CapCondition capCondition,
bool isMetadataPropertyCollection,
MetadataPropertyCollection metadataPropertyCollection,
DomainSpecificProcessingType domainSpecificProcessingType,
DomainSpecificConfig domainSpecificConfig,
string getDistinctValuesFieldName,
GroupBy groupBy,
bool forceFullGet)
{
CacheIndexInternal cacheIndexInternal = null;
byte[] extendedId = FormExtendedId(indexId, extendedIdSuffix);
//RelayMessage getMsg = new RelayMessage(typeId, primaryId, extendedId, MessageType.Get);
//storeContext.IndexStorageComponent.HandleMessage(getMsg);
Stream myStream;
ResourcePoolItem <MemoryStream> pooledStreamItem = null;
MemoryStream pooledStream;
try
{
int indexLevelGetSize =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId].
IndexCollection[indexName].PartialGetSizeInBytes;
bool isFullGet = forceFullGet ? true : (indexLevelGetSize <= 0 && storeContext.PartialGetLength <= 0);
if (isFullGet)
{
byte[] cacheBytes = BinaryStorageAdapter.Get(
storeContext.IndexStorageComponent,
typeId,
primaryId,
extendedId);
if (cacheBytes != null)
{
myStream = new MemoryStream(cacheBytes);
}
else
{
return(null);
}
}
else
{
int partialGetSize = indexLevelGetSize == 0 ? storeContext.PartialGetLength : indexLevelGetSize;
// get a memory stream from the memory pool
pooledStreamItem = storeContext.MemoryPool.GetItem();
pooledStream = pooledStreamItem.Item;
myStream = new SmartStream(
storeContext.IndexStorageComponent,
typeId,
primaryId,
extendedId,
partialGetSize,
pooledStream);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.PartialGetPerSec,
1);
}
if (myStream.Length > 0) //CacheIndex exists, this check is just for SmartStream
{
cacheIndexInternal = new CacheIndexInternal
{
InDeserializationContext =
new InDeserializationContext(maxItemsPerIndex,
indexName,
indexId,
typeId,
filter,
inclusiveFilter,
storeContext.TagHashCollection,
deserializeHeaderOnly,
getFilteredItems,
primarySortInfo,
localIdentityTagNames,
storeContext.StringHashCollection,
stringHashCodeDictionary,
indexCondition,
capCondition,
isMetadataPropertyCollection,
metadataPropertyCollection,
domainSpecificProcessingType,
domainSpecificConfig,
getDistinctValuesFieldName,
groupBy)
};
if (!isFullGet)
{
// skip the bdb entry header
myStream.Read(new byte[BdbHeaderSize], 0, BdbHeaderSize);
}
// This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal
// ** TBD - Should be removed later
if (LegacySerializationUtil.Instance.IsSupported(typeId))
{
cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream));
}
else
{
int version = myStream.ReadByte();
try
{
cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream), version);
}
catch (Exception ex)
{
LoggingUtil.Log.ErrorFormat(
"The deserialization has an exception: primary id : {0}, index id : {1}, extendedid : {2}, extendedIdSuffix : {3}, version : {4} info : {5}",
primaryId,
ByteArrayToString(indexId, 0),
ByteArrayToString(extendedId, 0),
extendedIdSuffix,
version,
ex.ToString());
if (myStream.Length > 0)
{
myStream.Seek(0, SeekOrigin.Begin);
byte[] ba = new byte[10];
myStream.Read(ba, 0, 10);
LoggingUtil.Log.ErrorFormat("The first 10 bytes of the stream are {0}",
ByteArrayToString(ba, 0));
}
throw;
}
}
// update SmartStream perf counters
if (!isFullGet)
{
SmartStream mySmartStream = (SmartStream)myStream;
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGet,
mySmartStream.DatabaseAccessTime);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGetBase,
1);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGet,
mySmartStream.Position);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGetBase,
1);
}
}
}
finally
{
// release the pooled stream
if (storeContext.MemoryPool != null && pooledStreamItem != null)
{
storeContext.MemoryPool.ReleaseItem(pooledStreamItem);
}
}
return(cacheIndexInternal);
}
