/// <summary>
/// Processes the specified message context.
/// </summary>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(MessageContext messageContext, IndexStoreContext storeContext)
{
// TBD : Support concurrent DeleteAllInType for different types
lock (LockingUtil.Instance.LockerObjects)
{
RelayMessage msg;
short typeId = messageContext.TypeId;
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId].IndexServerMode))
{
// Send DeleteAll to Data Store
short relatedTypeId;
if (!storeContext.TryGetRelatedIndexTypeId(typeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", typeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + typeId);
}
msg = new RelayMessage(relatedTypeId, 0, MessageType.DeleteAllInType);
storeContext.ForwarderComponent.HandleMessage(msg);
}
// Send DeleteAll to local Index storage
BinaryStorageAdapter.Clear(storeContext.IndexStorageComponent, typeId);
// Delete hash mapping entries in file
storeContext.RemoveType(typeId);
}
}
/// <summary>
/// Gets the metadata when its stored seperately.
/// </summary>
/// <param name="indexTypeMapping">The index type mapping.</param>
/// <param name="typeId">The type id.</param>
/// <param name="primaryId">The primary id.</param>
/// <param name="extendedId">The extended id.</param>
/// <param name="storeContext">The store context.</param>
/// <param name="metadata">The Metadata.</param>
/// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param>
internal static void GetMetadataStoredSeperately(IndexTypeMapping indexTypeMapping,
short typeId,
int primaryId,
byte[] extendedId,
IndexStoreContext storeContext,
out byte[] metadata,
out MetadataPropertyCollection metadataPropertyCollection)
{
metadata = null;
metadataPropertyCollection = null;
byte[] metadataBytes = BinaryStorageAdapter.Get(storeContext.IndexStorageComponent, typeId, primaryId, extendedId);
if (metadataBytes != null)
{
if (indexTypeMapping.IsMetadataPropertyCollection)
{
metadataPropertyCollection = GetMetadataPropertyCollection(metadataBytes);
}
else
{
metadata = metadataBytes;
}
}
}
/// <summary>
/// Processes the specified filtered index delete command.
/// </summary>
/// <param name="filteredIndexDeleteCommand">The filtered index delete command.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(
FilteredIndexDeleteCommand filteredIndexDeleteCommand,
MessageContext messageContext,
IndexStoreContext storeContext)
{
if (filteredIndexDeleteCommand != null)
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
Index indexInfo = indexTypeMapping.IndexCollection[filteredIndexDeleteCommand.TargetIndexName];
byte[] metadata = null;
MetadataPropertyCollection metadataPropertyCollection = null;
#region Get CacheIndexInternal for TargetIndexName
if (indexTypeMapping.MetadataStoredSeperately)
{
IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping,
messageContext.TypeId,
messageContext.PrimaryId,
filteredIndexDeleteCommand.IndexId,
storeContext,
out metadata,
out metadataPropertyCollection);
}
CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
filteredIndexDeleteCommand.PrimaryId,
filteredIndexDeleteCommand.IndexId,
indexInfo.ExtendedIdSuffix,
filteredIndexDeleteCommand.TargetIndexName,
0,
filteredIndexDeleteCommand.DeleteFilter,
false,
null,
false,
true,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null,
indexInfo.IsMetadataPropertyCollection,
metadataPropertyCollection,
DomainSpecificProcessingType.None,
null,
null,
null,
false);
#endregion
if (cacheIndexInternal != null)
{
#region Increment perf counters' number
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerFilterDeleteRequest,
messageContext.TypeId,
cacheIndexInternal.OutDeserializationContext.TotalCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerFilterDeleteRequest,
messageContext.TypeId,
cacheIndexInternal.OutDeserializationContext.ReadItemCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsFilteredPerFilterDeleteRequest,
messageContext.TypeId,
(cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count));
#endregion
#region Update VirtualCount
cacheIndexInternal.VirtualCount -= (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.Count);
#endregion
#region Save CacheIndexInternal to local storage since item which pass delete filter are pruned in it
byte[] extendedId = IndexServerUtils.FormExtendedId(filteredIndexDeleteCommand.IndexId,
indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix);
// compose a bdb entry header
bool isCompress = storeContext.GetCompressOption(messageContext.TypeId);
PayloadStorage bdbEntryHeader = new PayloadStorage
{
Compressed = isCompress,
TTL = -1,
LastUpdatedTicks = DateTime.Now.Ticks,
ExpirationTicks = -1,
Deactivated = false
};
BinaryStorageAdapter.Save(
storeContext.MemoryPool,
storeContext.IndexStorageComponent,
messageContext.TypeId,
filteredIndexDeleteCommand.PrimaryId,
extendedId,
bdbEntryHeader,
Serializer.Serialize <CacheIndexInternal>(cacheIndexInternal, isCompress, RelayMessage.RelayCompressionImplementation));
#endregion
#region Data store deletes
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
indexTypeMapping.IndexServerMode) &&
cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList != null &&
cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count > 0)
{
List <RelayMessage> dataStorageMessageList = new List <RelayMessage>();
short relatedTypeId;
if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
cacheIndexInternal.InternalItemList = cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList;
List <byte[]> fullDataIdList = DataTierUtil.GetFullDataIds(cacheIndexInternal.InDeserializationContext.IndexId,
cacheIndexInternal.InternalItemList,
indexTypeMapping.FullDataIdFieldList);
foreach (byte[] fullDataId in fullDataIdList)
{
if (fullDataId != null)
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
MessageType.Delete));
}
}
if (dataStorageMessageList.Count > 0)
{
storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList);
}
}
#endregion
}
}
}
/// <summary>
/// Processes the specified message context.
/// </summary>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(MessageContext messageContext, IndexStoreContext storeContext)
{
lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId))
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
List <RelayMessage> indexStorageMessageList = new List <RelayMessage>(indexTypeMapping.IndexCollection.Count);
List <RelayMessage> dataStorageMessageList = new List <RelayMessage>();
CacheIndexInternal internalIndex;
List <byte[]> fullDataIdList;
if (indexTypeMapping.MetadataStoredSeperately)
{
BinaryStorageAdapter.Delete(
storeContext.IndexStorageComponent,
messageContext.TypeId,
messageContext.PrimaryId,
messageContext.ExtendedId);
}
foreach (Index index in indexTypeMapping.IndexCollection)
{
internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
messageContext.ExtendedId,
index.ExtendedIdSuffix,
index.IndexName,
0,
null,
true,
null,
false,
false,
index.PrimarySortInfo,
index.LocalIdentityTagList,
index.StringHashCodeDictionary,
null,
indexTypeMapping.IndexCollection[index.IndexName].IsMetadataPropertyCollection,
null,
DomainSpecificProcessingType.None,
null,
null,
null,
true);
if (internalIndex != null)
{
#region Deletes messages for data store
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
indexTypeMapping.IndexServerMode))
{
fullDataIdList = DataTierUtil.GetFullDataIds(messageContext.ExtendedId,
internalIndex.InternalItemList,
indexTypeMapping.FullDataIdFieldList);
short relatedTypeId;
foreach (byte[] fullDataId in fullDataIdList)
{
if (fullDataId != null)
{
if (storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
MessageType.Delete));
}
else
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
}
}
}
#endregion
#region Delete messages for index store
BinaryStorageAdapter.Delete(
storeContext.IndexStorageComponent,
messageContext.TypeId,
messageContext.PrimaryId,
IndexServerUtils.FormExtendedId(messageContext.ExtendedId, index.ExtendedIdSuffix));
#endregion
}
}
if (dataStorageMessageList.Count > 0)
{
storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList);
}
}
}
/// <summary>
/// Gets the CacheIndexInternal.
/// </summary>
/// <param name="storeContext">The store context.</param>
/// <param name="typeId">The type id.</param>
/// <param name="primaryId">The primary id.</param>
/// <param name="indexId">The index id.</param>
/// <param name="extendedIdSuffix">The extended id suffix.</param>
/// <param name="indexName">Name of the index.</param>
/// <param name="maxItemsPerIndex">The maxItemsPerIndex.</param>
/// <param name="filter">The filter.</param>
/// <param name="inclusiveFilter">if set to <c>true</c> includes the items that pass the filter; otherwise , <c>false</c>.</param>
/// <param name="indexCondition">The index condition.</param>
/// <param name="deserializeHeaderOnly">if set to <c>true</c> if just CacheIndexInternal header is to be deserialized; otherwise, <c>false</c>.</param>
/// <param name="getFilteredItems">if set to <c>true</c> get filtered items; otherwise, <c>false</c>.</param>
/// <param name="primarySortInfo">The primary sort info.</param>
/// <param name="localIdentityTagNames">The local identity tag names.</param>
/// <param name="stringHashCodeDictionary">The string hash code dictionary.</param>
/// <param name="capCondition">The cap condition.</param>
/// <param name="isMetadataPropertyCollection">if set to <c>true</c> metadata represents a property collection; otherwise , <c>false</c>.</param>
/// <param name="metadataPropertyCollection">The MetadataPropertyCollection.</param>
/// <param name="domainSpecificProcessingType">The DomainSpecificProcessingType.</param>
/// <param name="domainSpecificConfig">The DomainSpecificConfig.</param>
/// <param name="getDistinctValuesFieldName">The distinct value field name.</param>
/// <param name="groupBy">The GroupBy clause.</param>
/// <param name="forceFullGet">indicate whether or not use full get.</param>
/// <returns>CacheIndexInternal</returns>
internal static CacheIndexInternal GetCacheIndexInternal(IndexStoreContext storeContext,
short typeId,
int primaryId,
byte[] indexId,
short extendedIdSuffix,
string indexName,
int maxItemsPerIndex,
Filter filter,
bool inclusiveFilter,
IndexCondition indexCondition,
bool deserializeHeaderOnly,
bool getFilteredItems,
PrimarySortInfo primarySortInfo,
List <string> localIdentityTagNames,
Dictionary <int, bool> stringHashCodeDictionary,
CapCondition capCondition,
bool isMetadataPropertyCollection,
MetadataPropertyCollection metadataPropertyCollection,
DomainSpecificProcessingType domainSpecificProcessingType,
DomainSpecificConfig domainSpecificConfig,
string getDistinctValuesFieldName,
GroupBy groupBy,
bool forceFullGet)
{
CacheIndexInternal cacheIndexInternal = null;
byte[] extendedId = FormExtendedId(indexId, extendedIdSuffix);
//RelayMessage getMsg = new RelayMessage(typeId, primaryId, extendedId, MessageType.Get);
//storeContext.IndexStorageComponent.HandleMessage(getMsg);
Stream myStream;
ResourcePoolItem <MemoryStream> pooledStreamItem = null;
MemoryStream pooledStream;
try
{
int indexLevelGetSize =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[typeId].
IndexCollection[indexName].PartialGetSizeInBytes;
bool isFullGet = forceFullGet ? true : (indexLevelGetSize <= 0 && storeContext.PartialGetLength <= 0);
if (isFullGet)
{
byte[] cacheBytes = BinaryStorageAdapter.Get(
storeContext.IndexStorageComponent,
typeId,
primaryId,
extendedId);
if (cacheBytes != null)
{
myStream = new MemoryStream(cacheBytes);
}
else
{
return(null);
}
}
else
{
int partialGetSize = indexLevelGetSize == 0 ? storeContext.PartialGetLength : indexLevelGetSize;
// get a memory stream from the memory pool
pooledStreamItem = storeContext.MemoryPool.GetItem();
pooledStream = pooledStreamItem.Item;
myStream = new SmartStream(
storeContext.IndexStorageComponent,
typeId,
primaryId,
extendedId,
partialGetSize,
pooledStream);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.PartialGetPerSec,
1);
}
if (myStream.Length > 0) //CacheIndex exists, this check is just for SmartStream
{
cacheIndexInternal = new CacheIndexInternal
{
InDeserializationContext =
new InDeserializationContext(maxItemsPerIndex,
indexName,
indexId,
typeId,
filter,
inclusiveFilter,
storeContext.TagHashCollection,
deserializeHeaderOnly,
getFilteredItems,
primarySortInfo,
localIdentityTagNames,
storeContext.StringHashCollection,
stringHashCodeDictionary,
indexCondition,
capCondition,
isMetadataPropertyCollection,
metadataPropertyCollection,
domainSpecificProcessingType,
domainSpecificConfig,
getDistinctValuesFieldName,
groupBy)
};
if (!isFullGet)
{
// skip the bdb entry header
myStream.Read(new byte[BdbHeaderSize], 0, BdbHeaderSize);
}
// This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal
// ** TBD - Should be removed later
if (LegacySerializationUtil.Instance.IsSupported(typeId))
{
cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream));
}
else
{
int version = myStream.ReadByte();
try
{
cacheIndexInternal.Deserialize(new CompactBinaryReader(myStream), version);
}
catch (Exception ex)
{
LoggingUtil.Log.ErrorFormat(
"The deserialization has an exception: primary id : {0}, index id : {1}, extendedid : {2}, extendedIdSuffix : {3}, version : {4} info : {5}",
primaryId,
ByteArrayToString(indexId, 0),
ByteArrayToString(extendedId, 0),
extendedIdSuffix,
version,
ex.ToString());
if (myStream.Length > 0)
{
myStream.Seek(0, SeekOrigin.Begin);
byte[] ba = new byte[10];
myStream.Read(ba, 0, 10);
LoggingUtil.Log.ErrorFormat("The first 10 bytes of the stream are {0}",
ByteArrayToString(ba, 0));
}
throw;
}
}
// update SmartStream perf counters
if (!isFullGet)
{
SmartStream mySmartStream = (SmartStream)myStream;
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGet,
mySmartStream.DatabaseAccessTime);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgDbGetPerPartialGetBase,
1);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGet,
mySmartStream.Position);
BerkeleyBinaryStorePerformanceCounters.Instance.IncrementCounter(
BerkeleyBinaryStorePerformanceCounterEnum.AvgBytesPerPartialGetBase,
1);
}
}
}
finally
{
// release the pooled stream
if (storeContext.MemoryPool != null && pooledStreamItem != null)
{
storeContext.MemoryPool.ReleaseItem(pooledStreamItem);
}
}
return(cacheIndexInternal);
}
/// <summary>
/// Processes the specified MetadataPropertyCommand.
/// </summary>
/// <param name="metadataPropertyCommand">The MetadataPropertyCommand.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(MetadataPropertyCommand metadataPropertyCommand, MessageContext messageContext, IndexStoreContext storeContext)
{
if (metadataPropertyCommand != null)
{
MetadataPropertyCollection metadataPropertyCollection = null;
byte[] metadata;
IndexTypeMapping indexTypeMapping = storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
CacheIndexInternal cacheIndexInternal = null;
#region Fetch MetadataPropertyCollection
if (indexTypeMapping.MetadataStoredSeperately)
{
IndexServerUtils.GetMetadataStoredSeperately(indexTypeMapping,
messageContext.TypeId,
metadataPropertyCommand.PrimaryId,
metadataPropertyCommand.IndexId,
storeContext,
out metadata,
out metadataPropertyCollection);
}
else
{
Index indexInfo = indexTypeMapping.IndexCollection[metadataPropertyCommand.TargetIndexName];
// Get CacheIndexInternal
cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
metadataPropertyCommand.PrimaryId,
metadataPropertyCommand.IndexId,
indexInfo.ExtendedIdSuffix,
metadataPropertyCommand.TargetIndexName,
0,
null,
false,
null,
true,
false,
null,
null,
null,
null,
true,
null,
DomainSpecificProcessingType.None,
null,
null,
null,
true);
if (cacheIndexInternal != null && cacheIndexInternal.MetadataPropertyCollection != null)
{
metadataPropertyCollection = cacheIndexInternal.MetadataPropertyCollection;
}
}
#endregion
#region Process MetadataPropertyCollection add/deletes
if (metadataPropertyCollection == null)
{
metadataPropertyCollection = new MetadataPropertyCollection();
}
metadataPropertyCollection.Process(metadataPropertyCommand.MetadataPropertyCollectionUpdate);
#endregion
#region Restore MetadataPropertyCollection back to storage
bool isCompress = storeContext.GetCompressOption(messageContext.TypeId);
byte[] extId = null;
byte[] byteArray = null;
if (indexTypeMapping.MetadataStoredSeperately)
{
extId = metadataPropertyCommand.ExtendedId;
byteArray = Serializer.Serialize(metadataPropertyCollection, isCompress);
}
else
{
if (cacheIndexInternal == null)
{
cacheIndexInternal = new CacheIndexInternal
{
InDeserializationContext = new InDeserializationContext(0,
metadataPropertyCommand.TargetIndexName,
metadataPropertyCommand.IndexId,
messageContext.TypeId,
null,
false,
null,
true,
false,
null,
null,
null,
null,
null,
null,
true,
null,
DomainSpecificProcessingType.None,
null,
null,
null)
};
}
//Restore CacheIndexInternal
cacheIndexInternal.MetadataPropertyCollection = metadataPropertyCollection;
extId = IndexServerUtils.FormExtendedId(metadataPropertyCommand.IndexId,
indexTypeMapping.IndexCollection[
cacheIndexInternal.InDeserializationContext.IndexName].
ExtendedIdSuffix);
byteArray = Serializer.Serialize(cacheIndexInternal, isCompress);
}
PayloadStorage bdbEntryHeader = new PayloadStorage
{
Compressed = isCompress,
TTL = -1,
LastUpdatedTicks = DateTime.Now.Ticks,
ExpirationTicks = -1,
Deactivated = false
};
BinaryStorageAdapter.Save(
storeContext.MemoryPool,
storeContext.IndexStorageComponent,
messageContext.TypeId,
metadataPropertyCommand.PrimaryId,
extId,
bdbEntryHeader,
byteArray);
#endregion
}
}
