/// <summary>
/// Processes the specified message context.
/// </summary>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(MessageContext messageContext, IndexStoreContext storeContext)
{
// TBD : Support concurrent DeleteAllInType for different types
lock (LockingUtil.Instance.LockerObjects)
{
RelayMessage msg;
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId].IndexServerMode))
{
// Send DeleteAll to Data Store
short relatedTypeId;
if(!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
msg = new RelayMessage(relatedTypeId, 0, MessageType.DeleteAllInType);
storeContext.ForwarderComponent.HandleMessage(msg);
}
// Send DeleteAll to local Index storage
msg = new RelayMessage(messageContext.TypeId, 0, MessageType.DeleteAllInType);
storeContext.IndexStorageComponent.HandleMessage(msg);
// Delete hash mapping entries in file
storeContext.RemoveType(messageContext.TypeId);
}
}
/// <summary>
/// Processes the specified message context.
/// </summary>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>Raw CacheIndexInternal</returns>
internal static byte[] Process(MessageContext messageContext, IndexStoreContext storeContext)
{
byte[] payloadByteArray = null;
RelayMessage getMsg = new RelayMessage
{
MessageType = MessageType.Get,
Id = IndexCacheUtils.GeneratePrimaryId(messageContext.ExtendedId),
ExtendedId = IndexServerUtils.FormExtendedId(messageContext.ExtendedId, 0) // Pull first of the multiple indexes
};
if (storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection.Contains(messageContext.TypeId))
{
getMsg.TypeId = messageContext.TypeId;
}
else if (storeContext.RelatedTypeIds.TryGetValue(messageContext.TypeId, out getMsg.TypeId))
{
}
else
{
LoggingUtil.Log.InfoFormat("Invalid TypeID for GetMessage {0}", messageContext.TypeId);
return payloadByteArray;
}
storeContext.IndexStorageComponent.HandleMessage(getMsg);
if (getMsg.Payload != null)
{
payloadByteArray = getMsg.Payload.ByteArray;
}
return payloadByteArray;
}
/// <summary>
/// Gets the data into resultItemList items.
/// </summary>
/// <param name="resultItemList">The result item list.</param>
/// <param name="storeContext">The store context.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="fullDataIdFieldList">The full data id field list.</param>
/// <param name="fullDataIdInfo">The full data id info.</param>
internal static void GetData(List<ResultItem> resultItemList,
IndexStoreContext storeContext,
MessageContext messageContext,
FullDataIdFieldList fullDataIdFieldList,
FullDataIdInfo fullDataIdInfo)
{
byte[] extendedId;
List<RelayMessage> dataStoreMessages = new List<RelayMessage>();
short relatedTypeId;
if (fullDataIdInfo != null && fullDataIdInfo.RelatedTypeName != null)
{
if (!storeContext.TryGetTypeId(fullDataIdInfo.RelatedTypeName, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", fullDataIdInfo.RelatedTypeName);
throw new Exception("Invalid RelatedTypeId for TypeId - " + fullDataIdInfo.RelatedTypeName);
}
}
else if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
if (resultItemList != null && resultItemList.Count > 0)
{
foreach (ResultItem resultItem in resultItemList)
{
extendedId = GetFullDataId(resultItem.IndexId,
resultItem,
fullDataIdInfo != null && fullDataIdInfo.RelatedTypeName != null ? fullDataIdInfo.FullDataIdFieldList : fullDataIdFieldList);
dataStoreMessages.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(extendedId),
extendedId,
MessageType.Get));
}
storeContext.ForwarderComponent.HandleMessages(dataStoreMessages);
int i = 0;
foreach (ResultItem resultItem in resultItemList)
{
if (dataStoreMessages[i].Payload != null)
{
resultItem.Data = dataStoreMessages[i].Payload.ByteArray;
}
else
{
LoggingUtil.Log.DebugFormat("Fetched Null Data for TypeId: {0}, IndexId: {1}, ItemId: {2}, FullDataId: {3}, PrimaryId: {4}",
relatedTypeId,
IndexCacheUtils.GetReadableByteArray(resultItem.IndexId),
IndexCacheUtils.GetReadableByteArray(resultItem.ItemId),
IndexCacheUtils.GetReadableByteArray(dataStoreMessages[i].ExtendedId),
IndexCacheUtils.GeneratePrimaryId(dataStoreMessages[i].ExtendedId)
);
}
i++;
}
}
}
public void GetDataItems(FullDataIdInfo info, bool excludeData, MessageContext messageContext, IndexStoreContext storeContext, BaseMultiIndexIdQueryResult queryResult)
{
if (excludeData == false)
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
DataTierUtil.GetData(queryResult.ResultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, info);
}
}
/// <summary>
/// Processes the specified cache index update.
/// </summary>
/// <param name="cacheIndexUpdate">The cache index update.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(CacheIndexUpdate cacheIndexUpdate, MessageContext messageContext, IndexStoreContext storeContext)
{
switch (cacheIndexUpdate.Command.CommandType)
{
case CommandType.FilteredIndexDelete:
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.FilterDelete,
messageContext.TypeId,
1);
FilteredIndexDeleteProcessor.Process(cacheIndexUpdate.Command as FilteredIndexDeleteCommand, messageContext, storeContext);
break;
}
}
/// <summary>
/// Processes the specified remote clustered intersection query.
/// </summary>
/// <param name="remoteClusteredIntersectionQuery">The remote clustered intersection query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>IntersectionQueryResult</returns>
internal static IntersectionQueryResult Process(RemoteClusteredIntersectionQuery remoteClusteredIntersectionQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
// increment performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.IndexLookupAvgPerRemoteClusteredIntersectionQuery,
messageContext.TypeId,
remoteClusteredIntersectionQuery.IndexIdList.Count);
VirtualClusteredIntersectionQuery query = new VirtualClusteredIntersectionQuery(
remoteClusteredIntersectionQuery.IndexIdList,
remoteClusteredIntersectionQuery.TargetIndexName,
storeContext.GetTypeName(messageContext.TypeId))
{
ExcludeData = remoteClusteredIntersectionQuery.ExcludeData,
GetIndexHeader = remoteClusteredIntersectionQuery.GetIndexHeader,
Filter = remoteClusteredIntersectionQuery.Filter,
intersectionQueryParamsMapping = remoteClusteredIntersectionQuery.IntersectionQueryParamsMapping,
PrimaryId = remoteClusteredIntersectionQuery.PrimaryId,
PrimaryIdList = remoteClusteredIntersectionQuery.PrimaryIdList
};
return RelayClient.Instance.SubmitQuery<VirtualClusteredIntersectionQuery, IntersectionQueryResult>(query);
}
internal SpanQueryResult Process(RemoteClusteredSpanQuery remoteClusteredSpanQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
// increment the performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.IndexLookupAvgPerRemoteClusteredSpanQuery,
messageContext.TypeId,
remoteClusteredSpanQuery.IndexIdList.Count);
// creating a spanQuery with copy ctor
VirtualSpanQuery query = new VirtualSpanQuery(remoteClusteredSpanQuery)
{
CacheTypeName = storeContext.GetTypeName(messageContext.TypeId),
// we get only reference at this point
ExcludeData = true
};
SpanQueryResult queryResult = RelayClient.Instance.SubmitQuery<VirtualSpanQuery, SpanQueryResult>(query);
GetDataItems(remoteClusteredSpanQuery.FullDataIdInfo, remoteClusteredSpanQuery.ExcludeData, messageContext, storeContext, queryResult);
return queryResult;
}
/// <summary>
/// Processes the get message.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessGetMessage(RelayMessage message, MessageContext messageContext)
{
byte[] payloadByteArray = GetProcessor.Process(messageContext, storeContext);
if (payloadByteArray != null)
{
bool compress = storeContext.GetCompressOption(message.TypeId);
message.Payload = new RelayPayload(message.TypeId, message.Id, payloadByteArray, compress);
}
else
{
LoggingUtil.Log.InfoFormat("Miss in CacheIndexStorage for Id : {0}, ExtendedId : ", message.Id, IndexCacheUtils.GetReadableByteArray(message.ExtendedId));
}
}
/// <summary>
/// Processes the specified message context.
/// </summary>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(MessageContext messageContext, IndexStoreContext storeContext)
{
lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId))
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
List<RelayMessage> dataStorageMessageList = new List<RelayMessage>();
List<RelayMessage> indexStorageMessageList = new List<RelayMessage>(indexTypeMapping.IndexCollection.Count);
CacheIndexInternal internalIndex;
List<byte[]> fullDataIdList;
if (indexTypeMapping.MetadataStoredSeperately)
{
indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId,
messageContext.PrimaryId,
messageContext.ExtendedId,
MessageType.Delete));
}
foreach (Index index in indexTypeMapping.IndexCollection)
{
internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
messageContext.ExtendedId,
index.ExtendedIdSuffix,
index.IndexName,
0,
null,
true,
null,
false,
false,
index.PrimarySortInfo,
index.LocalIdentityTagList,
index.StringHashCodeDictionary,
null);
if (internalIndex != null)
{
#region Deletes messages for data store
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
indexTypeMapping.IndexServerMode))
{
fullDataIdList = DataTierUtil.GetFullDataIds(messageContext.ExtendedId,
internalIndex.InternalItemList,
indexTypeMapping.FullDataIdFieldList);
short relatedTypeId;
foreach (byte[] fullDataId in fullDataIdList)
{
if (fullDataId != null)
{
if (storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
MessageType.Delete));
}
else
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
}
}
}
#endregion
#region Delete messages for index store
indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId,
messageContext.PrimaryId,
IndexServerUtils.FormExtendedId(messageContext.ExtendedId, index.ExtendedIdSuffix),
MessageType.Delete));
#endregion
}
}
if (dataStorageMessageList.Count > 0)
{
storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList);
}
if (indexStorageMessageList.Count > 0)
{
storeContext.IndexStorageComponent.HandleMessages(indexStorageMessageList);
}
}
}
/// <summary>
/// Processes the specified filtered index delete command.
/// </summary>
/// <param name="filteredIndexDeleteCommand">The filtered index delete command.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(
FilteredIndexDeleteCommand filteredIndexDeleteCommand,
MessageContext messageContext,
IndexStoreContext storeContext)
{
if (filteredIndexDeleteCommand != null)
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
Index indexInfo = indexTypeMapping.IndexCollection[filteredIndexDeleteCommand.TargetIndexName];
#region Get CacheIndexInternal for TargetIndexName
CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
filteredIndexDeleteCommand.PrimaryId,
filteredIndexDeleteCommand.IndexId,
indexInfo.ExtendedIdSuffix,
filteredIndexDeleteCommand.TargetIndexName,
0,
filteredIndexDeleteCommand.DeleteFilter,
false,
null,
false,
true,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null);
#endregion
if (cacheIndexInternal != null)
{
#region Increment perf counters' number
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerFilterDeleteRequest,
messageContext.TypeId,
cacheIndexInternal.OutDeserializationContext.TotalCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerFilterDeleteRequest,
messageContext.TypeId,
cacheIndexInternal.OutDeserializationContext.ReadItemCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsFilteredPerFilterDeleteRequest,
messageContext.TypeId,
(cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count));
#endregion
#region Update VirtualCount
cacheIndexInternal.VirtualCount -= (cacheIndexInternal.OutDeserializationContext.TotalCount - cacheIndexInternal.Count);
#endregion
#region Save CacheIndexInternal to local storage since item which pass delete filter are pruned in it
byte[] extendedId = IndexServerUtils.FormExtendedId(filteredIndexDeleteCommand.IndexId,
indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix);
RelayMessage indexStorageMessage = RelayMessage.GetSaveMessageForObject(messageContext.TypeId,
filteredIndexDeleteCommand.PrimaryId,
extendedId,
DateTime.Now,
cacheIndexInternal,
storeContext.GetCompressOption(messageContext.TypeId));
storeContext.IndexStorageComponent.HandleMessage(indexStorageMessage);
#endregion
#region Data store deletes
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
indexTypeMapping.IndexServerMode) &&
cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList != null &&
cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList.Count > 0)
{
List<RelayMessage> dataStorageMessageList = new List<RelayMessage>();
short relatedTypeId;
if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
cacheIndexInternal.InternalItemList = cacheIndexInternal.OutDeserializationContext.FilteredInternalItemList;
List<byte[]> fullDataIdList = DataTierUtil.GetFullDataIds(cacheIndexInternal.InDeserializationContext.IndexId,
cacheIndexInternal.InternalItemList,
indexTypeMapping.FullDataIdFieldList);
foreach (byte[] fullDataId in fullDataIdList)
{
if (fullDataId != null)
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
MessageType.Delete));
}
}
if (dataStorageMessageList.Count > 0)
{
storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList);
}
}
#endregion
}
}
}
/// <summary>
/// Processes the delete message.
/// </summary>
/// <param name="messageContext">The message context.</param>
private void ProcessDeleteMessage(MessageContext messageContext)
{
DeleteProcessor.Process(messageContext, storeContext);
}
/// <summary>
/// Processes the specified contains index query.
/// </summary>
/// <param name="containsIndexQuery">The contains index query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>ContainsIndexQueryResult</returns>
internal static ContainsIndexQueryResult Process(ContainsIndexQuery containsIndexQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
ContainsIndexQueryResult containsIndexQueryResult;
MultiItemResult multiItemResult = null;
byte[] metadata = null;
bool indexExists = false;
int indexSize = -1;
int virtualCount = -1;
try
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
#region Check TargetIndexName
if (!indexTypeMapping.IndexCollection.Contains(containsIndexQuery.TargetIndexName))
{
throw new Exception("Invalid TargetIndexName - " + containsIndexQuery.TargetIndexName);
}
#endregion
Index targetIndexInfo = indexTypeMapping.IndexCollection[containsIndexQuery.TargetIndexName];
List<CacheIndexInternal> internalCacheIndexList = new List<CacheIndexInternal>();
#region Get TargetIndex
CacheIndexInternal cacheIndexInternal = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
containsIndexQuery.IndexId,
targetIndexInfo.ExtendedIdSuffix,
containsIndexQuery.TargetIndexName,
0,
null,
true,
null,
false,
false,
targetIndexInfo.PrimarySortInfo,
targetIndexInfo.LocalIdentityTagList,
targetIndexInfo.StringHashCodeDictionary,
null);
#endregion
if (cacheIndexInternal != null)
{
internalCacheIndexList.Add(cacheIndexInternal);
indexExists = true;
indexSize = cacheIndexInternal.OutDeserializationContext.TotalCount;
virtualCount = cacheIndexInternal.VirtualCount;
// update the performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery,
messageContext.TypeId,
indexSize);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery,
messageContext.TypeId,
cacheIndexInternal.OutDeserializationContext.ReadItemCount);
int searchIndex;
IndexDataItem indexDataItem;
foreach (IndexItem queryIndexItem in containsIndexQuery.IndexItemList)
{
#region Search item in index
searchIndex = internalCacheIndexList[0].Search(queryIndexItem);
#endregion
if (searchIndex > -1)
{
if (multiItemResult == null)
{
multiItemResult = new MultiItemResult(containsIndexQuery.IndexId);
}
indexDataItem = new IndexDataItem(InternalItemAdapter.ConvertToIndexItem(internalCacheIndexList[0].GetItem(searchIndex),
internalCacheIndexList[0].InDeserializationContext));
#region Get extra tags
if (containsIndexQuery.TagsFromIndexes != null && containsIndexQuery.TagsFromIndexes.Count != 0)
{
foreach (string indexName in containsIndexQuery.TagsFromIndexes)
{
Index indexInfo = indexTypeMapping.IndexCollection[indexName];
CacheIndexInternal indexInternal =
IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
containsIndexQuery.IndexId,
indexInfo.ExtendedIdSuffix,
indexName,
0,
null,
true,
null,
false,
false,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null);
if (indexInternal != null)
{
// update the performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerContainsIndexQuery,
messageContext.TypeId,
indexInternal.OutDeserializationContext.TotalCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerContainsIndexQuery,
messageContext.TypeId,
indexInternal.OutDeserializationContext.ReadItemCount);
internalCacheIndexList.Add(indexInternal);
IndexServerUtils.GetTags(indexInternal, queryIndexItem, indexDataItem);
}
}
}
#endregion
multiItemResult.Add(indexDataItem);
}
}
#region Get data
if (!containsIndexQuery.ExcludeData && multiItemResult != null)
{
byte[] extendedId;
List<RelayMessage> dataStoreMessages = new List<RelayMessage>(multiItemResult.Count);
short relatedTypeId;
if (containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null)
{
if (!storeContext.TryGetTypeId(containsIndexQuery.FullDataIdInfo.RelatedTypeName, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedCacheTypeName - {0}", containsIndexQuery.FullDataIdInfo.RelatedTypeName);
throw new Exception("Invalid RelatedTypeId for TypeId - " + containsIndexQuery.FullDataIdInfo.RelatedTypeName);
}
}
else if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
foreach (IndexDataItem resultItem in multiItemResult)
{
extendedId = DataTierUtil.GetFullDataId(containsIndexQuery.IndexId,
resultItem,
containsIndexQuery.FullDataIdInfo != null && containsIndexQuery.FullDataIdInfo.RelatedTypeName != null ?
containsIndexQuery.FullDataIdInfo.FullDataIdFieldList :
indexTypeMapping.FullDataIdFieldList);
dataStoreMessages.Add(new RelayMessage(relatedTypeId, IndexCacheUtils.GeneratePrimaryId(extendedId), extendedId, MessageType.Get));
}
storeContext.ForwarderComponent.HandleMessages(dataStoreMessages);
int i = 0;
foreach (IndexDataItem resultItem in multiItemResult)
{
if (dataStoreMessages[i].Payload != null)
{
resultItem.Data = dataStoreMessages[i].Payload.ByteArray;
}
i++;
}
}
#endregion
#region Get metadata
if (containsIndexQuery.GetMetadata)
{
metadata = IndexServerUtils.GetQueryMetadata(internalCacheIndexList,
indexTypeMapping,
messageContext.TypeId,
messageContext.PrimaryId,
containsIndexQuery.IndexId,
storeContext);
}
#endregion
}
containsIndexQueryResult = new ContainsIndexQueryResult(multiItemResult, metadata, indexSize, indexExists, virtualCount, null);
}
catch (Exception ex)
{
containsIndexQueryResult = new ContainsIndexQueryResult(null, null, -1, false, -1, ex.Message);
LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing ContainsIndexQuery : {1}", messageContext.TypeId, ex);
}
return containsIndexQueryResult;
}
/// <summary>
/// Handles the message.
/// </summary>
/// <param name="message">The message.</param>
public void HandleMessage(RelayMessage message)
{
MessageContext messageContext = new MessageContext
{
PrimaryId = message.Id,
TypeId = message.TypeId,
ExtendedId = message.ExtendedId,
RelayTTL = message.RelayTTL,
SourceZone = message.SourceZone,
AddressHistory = message.AddressHistory
};
switch (message.MessageType)
{
case MessageType.Query:
ProcessQueryMessage(message, messageContext);
break;
case MessageType.Save:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.Save,
messageContext.TypeId,
1);
ProcessSaveMessage(message, messageContext);
break;
case MessageType.Update:
ProcessUpdateMessage(message, messageContext);
break;
case MessageType.Delete:
ProcessDeleteMessage(messageContext);
break;
case MessageType.DeleteAll:
LoggingUtil.Log.InfoFormat("CacheIndexV3Store does NOT support Message Type {0}", message.MessageType);
break;
case MessageType.DeleteAllInType:
ProcessDeleteAllInType(messageContext);
break;
case MessageType.DeleteInAllTypes:
LoggingUtil.Log.InfoFormat("CacheIndexV3Store does NOT support Message Type {0}", message.MessageType);
// do not throw exception - we can safely ignore this message type
break;
case MessageType.Get:
ProcessGetMessage(message, messageContext);
break;
default:
LoggingUtil.Log.ErrorFormat("Unknown Message Type {0}", message.MessageType);
break;
}
}
/// <summary>
/// Processes the query message.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessQueryMessage(RelayMessage message, MessageContext messageContext)
{
switch (message.QueryId)
{
case (byte)QueryTypes.PagedTaggedIndexQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.PagedIndexQuery,
messageContext.TypeId,
1);
ProcessPagedIndexQuery(message, messageContext);
break;
case (byte)QueryTypes.RemoteClusteredPagedIndexQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.RemoteClusteredPagedIndexQuery,
messageContext.TypeId,
1);
ProcessRemoteClusteredPagedIndexQuery(message, messageContext);
break;
case (byte)QueryTypes.ContainsIndexQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.ContainsIndexQuery,
messageContext.TypeId,
1);
ProcessContainsIndexQuery(message, messageContext);
break;
case (byte)QueryTypes.FirstLastQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.FirstLastQuery,
messageContext.TypeId,
1);
ProcessFirstLastQuery(message, messageContext);
break;
case (byte)QueryTypes.GetRangeQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.GetRangeQuery,
messageContext.TypeId,
1);
ProcessGetRangeQuery(message, messageContext);
break;
case (byte)QueryTypes.RandomQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.RandomQuery,
messageContext.TypeId,
1);
ProcessRandomQuery(message, messageContext);
break;
case (byte)QueryTypes.IntersectionQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.IntersectionQuery,
messageContext.TypeId,
1);
ProcessIntersectionQuery(message, messageContext);
break;
case (byte)QueryTypes.RemoteClusteredIntersectionQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.RemoteClusteredIntersectionQuery,
messageContext.TypeId,
1);
ProcessRemoteClusteredIntersectionQuery(message, messageContext);
break;
case (byte)QueryTypes.SpanQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.SpanQuery,
messageContext.TypeId,
1);
ProcessSpanQuery(message, messageContext);
break;
case (byte)QueryTypes.RemoteClusteredSpanQuery:
// increment performance counter
PerformanceCounters.Instance.IncrementCounter(
PerformanceCounterEnum.RemoteClusteredSpanQuery,
messageContext.TypeId,
1);
ProcessRemoteClusteredSpanQuery(message, messageContext);
break;
default:
throw new Exception("QueryId +" + (int)message.QueryId + "NOT supported");
}
}
/// <summary>
/// Processes the specified random query.
/// </summary>
/// <param name="randomQuery">The random query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>RandomQueryResult</returns>
internal static RandomQueryResult Process(RandomQuery randomQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
RandomQueryResult randomQueryResult;
List<ResultItem> resultItemList = null;
bool indexExists = false;
int indexSize = -1;
int virtualCount = -1;
byte[] metadata = null;
try
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
#region Validate Query
ValidateQuery(indexTypeMapping, randomQuery);
#endregion
Index targetIndexInfo = indexTypeMapping.IndexCollection[randomQuery.TargetIndexName];
#region Prepare Result
#region Extract index and apply criteria
CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
randomQuery.IndexId,
targetIndexInfo.ExtendedIdSuffix,
randomQuery.TargetIndexName,
0,
randomQuery.Filter,
true,
null,
false,
false,
targetIndexInfo.PrimarySortInfo,
targetIndexInfo.LocalIdentityTagList,
targetIndexInfo.StringHashCodeDictionary,
null);
#endregion
if (targetIndex != null)
{
indexSize = targetIndex.OutDeserializationContext.TotalCount;
virtualCount = targetIndex.VirtualCount;
indexExists = true;
// update performance counters
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerRandomQuery,
messageContext.TypeId,
indexSize);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerRandomQuery,
messageContext.TypeId,
targetIndex.OutDeserializationContext.ReadItemCount);
#region Populate resultLists
int indexItemCount = targetIndex.Count;
int resultListCount = Math.Min(randomQuery.Count, indexItemCount);
IEnumerable<int> itemPositionList = Algorithm.RandomSubset(new Random(), 0, indexItemCount - 1, resultListCount);
resultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, itemPositionList);
#endregion
#region Get data
if (!randomQuery.ExcludeData)
{
DataTierUtil.GetData(resultItemList, storeContext, messageContext, indexTypeMapping.FullDataIdFieldList, randomQuery.FullDataIdInfo);
}
#endregion
#region Get metadata
if (randomQuery.GetMetadata)
{
metadata = IndexServerUtils.GetQueryMetadata(new List<CacheIndexInternal>(1) { targetIndex },
indexTypeMapping,
messageContext.TypeId,
messageContext.PrimaryId,
randomQuery.IndexId,
storeContext);
}
#endregion
#endregion
}
randomQueryResult = new RandomQueryResult(indexExists, indexSize, metadata, resultItemList, virtualCount, null);
}
catch (Exception ex)
{
randomQueryResult = new RandomQueryResult(false, -1, null, null, -1, ex.Message);
LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing RandomQuery : {1}", messageContext.TypeId, ex);
}
return randomQueryResult;
}
/// <summary>
/// Processes the update message.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessUpdateMessage(RelayMessage message, MessageContext messageContext)
{
lock (LockingUtil.Instance.GetLock(message.Id))
{
try
{
CacheIndexUpdate cacheIndexUpdate = message.GetObject<CacheIndexUpdate>();
UpdateProcessor.Process(cacheIndexUpdate, messageContext, storeContext);
}
catch (Exception ex)
{
LoggingUtil.Log.ErrorFormat("TypeID {0} -- Error processing update message : {1}", message.TypeId, ex);
throw new Exception("Error processing update message");
}
}
}
/// <summary>
/// Processes the contains index query.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessContainsIndexQuery(RelayMessage message, MessageContext messageContext)
{
ContainsIndexQueryResult containsIndexQueryResult = ContainsQueryProcessor.Process(message.GetQueryObject<ContainsIndexQuery>(),
messageContext, storeContext);
bool compressOption = storeContext.GetCompressOption(message.TypeId);
message.Payload = new RelayPayload(message.TypeId,
message.Id,
Serializer.Serialize(containsIndexQueryResult, compressOption),
compressOption);
}
/// <summary>
/// Processes the first last query.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessFirstLastQuery(RelayMessage message, MessageContext messageContext)
{
FirstLastQueryResult firstLastQueryResult = FirstLastQueryProcessor.Process(message.GetQueryObject<FirstLastQuery>(),
messageContext, storeContext);
bool compressOption = storeContext.GetCompressOption(message.TypeId);
message.Payload = new RelayPayload(message.TypeId,
message.Id,
Serializer.Serialize(firstLastQueryResult, compressOption),
compressOption);
}
/// <summary>
/// Processes the remote clustered intersection query.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessRemoteClusteredIntersectionQuery(RelayMessage message, MessageContext messageContext)
{
IntersectionQueryResult intersectionQueryResult = RemoteClusteredIntersectionQueryProcessor.Process(message.GetQueryObject<RemoteClusteredIntersectionQuery>(),
messageContext,
storeContext);
AttachIntersectionQueryPayload(message, intersectionQueryResult);
}
/// <summary>
/// Processes the specified cache index.
/// </summary>
/// <param name="cacheIndex">Index of the cache.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
internal static void Process(CacheIndex cacheIndex, MessageContext messageContext, IndexStoreContext storeContext)
{
lock (LockingUtil.Instance.GetLock(messageContext.PrimaryId))
{
try
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
#region Extract CacheIndex and Validate from incoming message
ValidateSave(cacheIndex);
#endregion
#region Log CacheIndex before processing
StringBuilder dbgIndexInfo = null;
if (LoggingUtil.Log.IsDebugEnabled)
{
dbgIndexInfo = new StringBuilder();
dbgIndexInfo.Append("TypeId=").Append(messageContext.TypeId).Append(Environment.NewLine);
dbgIndexInfo.Append(IndexServerUtils.GetPrintableCacheIndex(cacheIndex,
storeContext.TagHashCollection,
messageContext.TypeId));
}
#endregion
#region Init vars
List<RelayMessage> indexStorageMessageList = new List<RelayMessage>();
List<RelayMessage> dataStorageMessageList = new List<RelayMessage>();
List<CacheIndexInternal> internalIndexList = new List<CacheIndexInternal>();
List<IndexItem> cappedDeleteItemList = new List<IndexItem>();
CacheIndexInternal internalIndex;
#endregion
if (cacheIndex.IndexVirtualCountMapping == null)
{
#region Save Items
#region Extract CacheIndexInternal from index storage
if (cacheIndex.TargetIndexName == null) //Save to multiple indexes
{
#region Get CacheIndexInternal for multiple indexes
foreach (KeyValuePair<string /*IndexName*/, List<string> /*TagNameList*/> kvp in cacheIndex.IndexTagMapping)
{
Index indexInfo = indexTypeMapping.IndexCollection[kvp.Key];
internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
cacheIndex.PrimaryId,
cacheIndex.IndexId,
indexInfo.ExtendedIdSuffix,
kvp.Key,
0,
null,
true,
null,
false,
false,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null);
if (internalIndex != null)
{
// update performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumberOfItemsInIndexPerSave,
messageContext.TypeId,
internalIndex.OutDeserializationContext.TotalCount);
}
if (internalIndex == null || cacheIndex.ReplaceFullIndex) //CacheIndexInternal does not exists or is to be discarded
{
internalIndex = new CacheIndexInternal
{
InDeserializationContext = new InDeserializationContext
{
TypeId = messageContext.TypeId,
TagHashCollection = storeContext.TagHashCollection,
IndexId = cacheIndex.IndexId,
IndexName = kvp.Key,
InclusiveFilter = true,
PrimarySortInfo = indexInfo.PrimarySortInfo,
LocalIdentityTagNames = indexInfo.LocalIdentityTagList,
StringHashCollection = storeContext.StringHashCollection,
StringHashCodeDictionary = indexInfo.StringHashCodeDictionary
}
};
}
internalIndexList.Add(internalIndex);
}
#endregion
}
else //Save to single index
{
#region Get CacheIndexInternal for TargetIndexName
Index indexInfo = indexTypeMapping.IndexCollection[cacheIndex.TargetIndexName];
internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
cacheIndex.PrimaryId,
cacheIndex.IndexId,
indexInfo.ExtendedIdSuffix,
cacheIndex.TargetIndexName,
0,
null,
true,
null,
false,
false,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null);
if (internalIndex != null)
{
// update performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumberOfItemsInIndexPerSave,
messageContext.TypeId,
internalIndex.OutDeserializationContext.TotalCount);
}
if (internalIndex == null || cacheIndex.ReplaceFullIndex) //CacheIndexInternal does not exists or is to be discarded
{
internalIndex = new CacheIndexInternal
{
InDeserializationContext = new InDeserializationContext
{
TypeId = messageContext.TypeId,
TagHashCollection = storeContext.TagHashCollection,
IndexId = cacheIndex.IndexId,
IndexName = cacheIndex.TargetIndexName,
InclusiveFilter = true,
PrimarySortInfo = indexInfo.PrimarySortInfo,
LocalIdentityTagNames = indexInfo.LocalIdentityTagList,
StringHashCollection = storeContext.StringHashCollection,
StringHashCodeDictionary = indexInfo.StringHashCodeDictionary
}
};
}
internalIndexList.Add(internalIndex);
#endregion
}
#endregion
#region Log CacheIndexInternals before save
if (LoggingUtil.Log.IsDebugEnabled && dbgIndexInfo != null)
{
dbgIndexInfo.Append(Environment.NewLine).Append(string.Format("BEFORE SAVE {0}",
IndexServerUtils.GetPrintableCacheIndexInternalList(
internalIndexList,
storeContext.TagHashCollection,
messageContext.TypeId)));
}
#endregion
#region Process Delete and Add List
try
{
#region Process Delete List
if (cacheIndex.DeleteList.Count > 0 && !cacheIndex.ReplaceFullIndex)
{
ProcessDeleteList(internalIndexList, cacheIndex.DeleteList, messageContext.TypeId);
}
#endregion
#region Process Add List
if (cacheIndex.AddList.Count > 0 || cacheIndex.UpdateMetadata)
{
ProcessAddList(internalIndexList, cappedDeleteItemList, cacheIndex, storeContext, indexTypeMapping);
}
#endregion
}
catch
{
LoggingUtil.Log.Debug(IndexServerUtils.GetPrintableCacheIndexInternalList(internalIndexList, storeContext.TagHashCollection, messageContext.TypeId));
throw;
}
#endregion
#region Log CacheIndexInternals after save
if (LoggingUtil.Log.IsDebugEnabled && dbgIndexInfo != null)
{
dbgIndexInfo.Append(Environment.NewLine).Append(string.Format("AFTER SAVE {0}",
IndexServerUtils.GetPrintableCacheIndexInternalList(internalIndexList,
storeContext.TagHashCollection,
messageContext.TypeId)));
}
#endregion
#region Data store relay messages for deletes and saves
if (DataTierUtil.ShouldForwardToDataTier(messageContext.RelayTTL,
messageContext.SourceZone,
storeContext.MyZone,
indexTypeMapping.IndexServerMode) && !cacheIndex.PreserveData)
{
byte[] fullDataId;
short relatedTypeId;
if (!storeContext.TryGetRelatedIndexTypeId(messageContext.TypeId, out relatedTypeId))
{
LoggingUtil.Log.ErrorFormat("Invalid RelatedTypeId for TypeId - {0}", messageContext.TypeId);
throw new Exception("Invalid RelatedTypeId for TypeId - " + messageContext.TypeId);
}
#region Delete Messages
foreach (IndexItem indexItem in cacheIndex.DeleteList)
{
fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexItem, indexTypeMapping.FullDataIdFieldList);
if (fullDataId != null)
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
MessageType.Delete));
}
}
#endregion
#region Save Messages
foreach (IndexDataItem indexDataItem in cacheIndex.AddList)
{
fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexDataItem, indexTypeMapping.FullDataIdFieldList);
if (fullDataId != null)
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
DateTime.Now,
indexDataItem.Data ?? new byte[0],
storeContext.GetCompressOption(messageContext.TypeId),
MessageType.Save));
if (indexDataItem.Data == null || indexDataItem.Data.Length == 0)
{
LoggingUtil.Log.WarnFormat("Saving null data for TypeId: {0}, IndexId: {1}, ItemId: {2}, FullDataId: {3}, PrimaryId: {4}",
relatedTypeId,
IndexCacheUtils.GetReadableByteArray(cacheIndex.IndexId),
IndexCacheUtils.GetReadableByteArray(indexDataItem.ItemId),
IndexCacheUtils.GetReadableByteArray(fullDataId),
IndexCacheUtils.GeneratePrimaryId(fullDataId));
}
}
}
#endregion
#region Capped Item Delete Messages
foreach (IndexItem indexItem in cappedDeleteItemList)
{
fullDataId = DataTierUtil.GetFullDataId(cacheIndex.IndexId, indexItem, indexTypeMapping.FullDataIdFieldList);
if (fullDataId != null)
{
dataStorageMessageList.Add(new RelayMessage(relatedTypeId,
IndexCacheUtils.GeneratePrimaryId(fullDataId),
fullDataId,
MessageType.Delete));
}
}
#endregion
#region Send relay mesaages to data store
if (dataStorageMessageList.Count > 0)
{
storeContext.ForwarderComponent.HandleMessages(dataStorageMessageList);
}
#endregion
}
#endregion
#endregion
if (dbgIndexInfo != null)
{
LoggingUtil.Log.Debug(dbgIndexInfo.ToString());
}
}
else
{
#region Update Virtual Count
foreach (KeyValuePair<string /*IndexName*/, int /*VirtualCount*/> kvp in cacheIndex.IndexVirtualCountMapping)
{
Index indexInfo = indexTypeMapping.IndexCollection[kvp.Key];
internalIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
cacheIndex.PrimaryId,
cacheIndex.IndexId,
indexInfo.ExtendedIdSuffix,
kvp.Key,
0,
null,
true,
null,
true,
false,
indexInfo.PrimarySortInfo,
indexInfo.LocalIdentityTagList,
indexInfo.StringHashCodeDictionary,
null);
if (internalIndex == null)
{
internalIndex = new CacheIndexInternal
{
InDeserializationContext = new InDeserializationContext
{
TypeId = messageContext.TypeId,
TagHashCollection = storeContext.TagHashCollection,
IndexId = cacheIndex.IndexId,
IndexName = kvp.Key,
InclusiveFilter = true,
DeserializeHeaderOnly = true,
PrimarySortInfo = indexInfo.PrimarySortInfo,
LocalIdentityTagNames = indexInfo.LocalIdentityTagList,
StringHashCollection = storeContext.StringHashCollection,
StringHashCodeDictionary = indexInfo.StringHashCodeDictionary
}
};
}
else
{
// update performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumberOfItemsInIndexPerSave,
messageContext.TypeId,
internalIndex.OutDeserializationContext.TotalCount);
}
internalIndex.VirtualCount = kvp.Value;
internalIndexList.Add(internalIndex);
}
#endregion
}
#region Index storage relay messages for each CacheIndexInternal
#region Metadata
if (indexTypeMapping.MetadataStoredSeperately && cacheIndex.UpdateMetadata)
{
indexStorageMessageList.Add(new RelayMessage(messageContext.TypeId,
cacheIndex.PrimaryId,
cacheIndex.IndexId,
DateTime.Now,
cacheIndex.Metadata ?? new byte[0],
storeContext.GetCompressOption(messageContext.TypeId),
MessageType.Save));
}
#endregion
#region Index(es)
byte[] payload;
CompactBinaryWriter writer;
RelayMessage indexStorageMessage;
byte[] extendedId;
foreach (CacheIndexInternal cacheIndexInternal in internalIndexList)
{
extendedId = IndexServerUtils.FormExtendedId(
cacheIndex.IndexId,
indexTypeMapping.IndexCollection[cacheIndexInternal.InDeserializationContext.IndexName].ExtendedIdSuffix);
// This mess is required until Moods 2.0 migrated to have IVersionSerializable version of CacheIndexInternal
// ** TBD - Should be removed later
if (LegacySerializationUtil.Instance.IsSupported(messageContext.TypeId))
{
writer = new CompactBinaryWriter(new BinaryWriter(new MemoryStream()));
cacheIndexInternal.Serialize(writer);
payload = new byte[writer.BaseStream.Length];
writer.BaseStream.Position = 0;
writer.BaseStream.Read(payload, 0, payload.Length);
indexStorageMessage = new RelayMessage(messageContext.TypeId,
cacheIndex.PrimaryId,
extendedId,
DateTime.Now,
payload,
storeContext.GetCompressOption(messageContext.TypeId),
MessageType.Save);
}
else
{
indexStorageMessage = RelayMessage.GetSaveMessageForObject(messageContext.TypeId,
cacheIndex.PrimaryId,
extendedId,
DateTime.Now,
cacheIndexInternal,
storeContext.GetCompressOption(messageContext.TypeId));
}
indexStorageMessageList.Add(indexStorageMessage);
}
#endregion
#region Send relay mesaages to index storage
storeContext.IndexStorageComponent.HandleMessages(indexStorageMessageList);
#endregion
#endregion
}
catch (Exception ex)
{
LoggingUtil.Log.DebugFormat("CacheIndex: {0}", IndexServerUtils.GetPrintableCacheIndex(cacheIndex, storeContext.TagHashCollection, messageContext.TypeId));
throw new Exception("TypeId " + messageContext.TypeId + " -- Error processing save message.", ex);
}
}
}
/// <summary>
/// Processes the specified intersection query.
/// </summary>
/// <param name="intersectionQuery">The intersection query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>IntersectionQueryResult</returns>
internal static IntersectionQueryResult Process(IntersectionQuery intersectionQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
//Fetch each index (assume all indexes are local) and perform intersection and return the results
IntersectionQueryResult intersectionQueryResult;
List<IndexDataItem> resultItemList = null;
Dictionary<byte[], IndexHeader> indexIdIndexHeaderMapping = new Dictionary<byte[], IndexHeader>(new ByteArrayEqualityComparer());
List<string> localIdentityTagNames = null;
bool isTagPrimarySort = false;
string sortFieldName = null;
List<SortOrder> sortOrderList = null;
StringBuilder exceptionInfo = new StringBuilder();
try
{
if (intersectionQuery.IndexIdList.Count > 0)
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
ValidateQuery(indexTypeMapping, intersectionQuery);
#region Set sort vars
Index targetIndexInfo = indexTypeMapping.IndexCollection[intersectionQuery.TargetIndexName];
localIdentityTagNames = indexTypeMapping.IndexCollection[intersectionQuery.TargetIndexName].LocalIdentityTagList;
bool sortFieldPartOfLocalId = IsSortFieldPartOfLocalId(localIdentityTagNames, targetIndexInfo.PrimarySortInfo);
TagSort itemIdTagSort = new TagSort("ItemId", false, new SortOrder(DataType.Int32, SortBy.ASC));
if (!sortFieldPartOfLocalId)
{
//Set sort vars
isTagPrimarySort = itemIdTagSort.IsTag;
sortFieldName = itemIdTagSort.TagName;
sortOrderList = new List<SortOrder>(1) { itemIdTagSort.SortOrder };
}
else
{
isTagPrimarySort = targetIndexInfo.PrimarySortInfo.IsTag;
sortFieldName = targetIndexInfo.PrimarySortInfo.FieldName;
sortOrderList = targetIndexInfo.PrimarySortInfo.SortOrderList;
}
#endregion
#region Fetch CacheIndexInternal and Intersect
CacheIndexInternal targetIndex;
CacheIndexInternal resultCacheIndexInternal = null;
IntersectionQueryParams indexIdParam;
byte[] indexId;
for (int i = 0; i < intersectionQuery.IndexIdList.Count; i++)
{
#region Extract index and apply criteria
indexId = intersectionQuery.IndexIdList[i];
indexIdParam = intersectionQuery.GetIntersectionQueryParamForIndexId(indexId);
targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
(intersectionQuery.PrimaryIdList != null && i < intersectionQuery.PrimaryIdList.Count) ?
intersectionQuery.PrimaryIdList[i] :
IndexCacheUtils.GeneratePrimaryId(indexId),
indexId,
targetIndexInfo.ExtendedIdSuffix,
intersectionQuery.TargetIndexName,
0,
indexIdParam.Filter,
true,
null,
false,
false,
targetIndexInfo.PrimarySortInfo,
targetIndexInfo.LocalIdentityTagList,
targetIndexInfo.StringHashCodeDictionary,
null);
#endregion
if (targetIndex != null)
{
if (targetIndex.Count < 0)
{
// No items in one of the indexes. Stop Interestion !!
resultCacheIndexInternal = null;
break;
}
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerIntersectionQuery,
messageContext.TypeId,
targetIndex.OutDeserializationContext.TotalCount);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerIntersectionQuery,
messageContext.TypeId,
targetIndex.OutDeserializationContext.ReadItemCount);
if (!sortFieldPartOfLocalId)
{
//Need to sort indexes by item id
targetIndex.Sort(itemIdTagSort);
}
#region Intersection
if (resultCacheIndexInternal == null)
{
// No need to perform intersection for first index
resultCacheIndexInternal = targetIndex;
}
else
{
IntersectionAlgo.Intersect(isTagPrimarySort,
sortFieldName,
localIdentityTagNames,
sortOrderList,
resultCacheIndexInternal.InternalItemList,
targetIndex.InternalItemList);
if (resultCacheIndexInternal == null || resultCacheIndexInternal.Count < 1)
{
// Unable to fetch one of the indexes. Stop Interestion !!
resultCacheIndexInternal = null;
indexIdIndexHeaderMapping = null;
break;
}
}
#endregion
}
else
{
// Unable to fetch one of the indexes. Stop Interestion !!
resultCacheIndexInternal = null;
indexIdIndexHeaderMapping = null;
break;
}
#region Get MetaData
if (intersectionQuery.GetIndexHeader)
{
if (!indexIdIndexHeaderMapping.ContainsKey(indexId))
{
indexIdIndexHeaderMapping.Add(indexId,
GetIndexHeader(targetIndex, indexTypeMapping, messageContext.TypeId, IndexCacheUtils.GeneratePrimaryId(indexId), storeContext));
}
}
#endregion
}
if (resultCacheIndexInternal != null && resultCacheIndexInternal.Count > 0)
{
resultItemList = CacheIndexInternalAdapter.GetIndexDataItemList(resultCacheIndexInternal, 1, int.MaxValue);
}
#endregion
#region Get data
if (!intersectionQuery.ExcludeData && resultItemList != null && resultItemList.Count > 0)
{
DataTierUtil.GetData(resultItemList,
storeContext,
messageContext,
indexTypeMapping.FullDataIdFieldList,
intersectionQuery.FullDataIdInfo);
}
#endregion
}
intersectionQueryResult = new IntersectionQueryResult(resultItemList,
indexIdIndexHeaderMapping,
localIdentityTagNames,
isTagPrimarySort,
sortFieldName,
sortOrderList,
exceptionInfo.ToString());
// update performance counter
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.IndexLookupAvgPerIntersectionQuery,
messageContext.TypeId,
intersectionQuery.IndexIdList.Count);
}
catch (Exception ex)
{
exceptionInfo.Append(" | " + ex.Message);
intersectionQueryResult = new IntersectionQueryResult(null, null, null, false, null, null, exceptionInfo.ToString());
LoggingUtil.Log.ErrorFormat("TypeId {0} -- Error processing IntersectionQuery : {1}", messageContext.TypeId, ex);
}
return intersectionQueryResult;
}
/// <summary>
/// Processes the paged index query.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessPagedIndexQuery(RelayMessage message, MessageContext messageContext)
{
PagedIndexQueryResult pagedIndexQueryResult = PagedQueryProcessor.Instance.Process(message.GetQueryObject<PagedIndexQuery>(),
messageContext, storeContext);
bool compressOption = storeContext.GetCompressOption(message.TypeId);
message.Payload = new RelayPayload(message.TypeId,
message.Id,
Serializer.Serialize(pagedIndexQueryResult, compressOption),
compressOption);
}
/// <summary>
/// Processes the save message.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="messageContext">The message context.</param>
private void ProcessSaveMessage(RelayMessage message, MessageContext messageContext)
{
SaveProcessor.Process(message.GetObject<CacheIndex>(), messageContext, storeContext);
}
/// <summary>
/// Process a RemoteClusteredSpanQuery
/// </summary>
/// <param name="message"></param>
/// <param name="messageContext"></param>
private void ProcessRemoteClusteredSpanQuery(RelayMessage message, MessageContext messageContext)
{
SpanQueryResult spanQueryResult = RemoteClusteredSpanQueryProcessor.Instance.Process(
message.GetQueryObject<RemoteClusteredSpanQuery>(),
messageContext,
storeContext);
bool compressOption = storeContext.GetCompressOption(message.TypeId);
message.Payload = new RelayPayload(message.TypeId,
message.Id,
Serializer.Serialize(spanQueryResult, compressOption),
compressOption);
}
/// <summary>
/// Processes the specified first last query.
/// </summary>
/// <param name="firstLastQuery">The first last query.</param>
/// <param name="messageContext">The message context.</param>
/// <param name="storeContext">The store context.</param>
/// <returns>FirstLastQueryResult</returns>
internal static FirstLastQueryResult Process(FirstLastQuery firstLastQuery, MessageContext messageContext, IndexStoreContext storeContext)
{
FirstLastQueryResult firstLastQueryResult;
List<ResultItem> firstPageResultItemList = null;
List<ResultItem> lastPageResultItemList = null;
bool indexExists = false;
int indexSize = -1;
int virtualCount = -1;
byte[] metadata = null;
try
{
IndexTypeMapping indexTypeMapping =
storeContext.StorageConfiguration.CacheIndexV3StorageConfig.IndexTypeMappingCollection[messageContext.TypeId];
#region Validate Query
ValidateQuery(indexTypeMapping, firstLastQuery);
#endregion
int maxItemsPerIndex = ((firstLastQuery.FirstPageSize > 0) && (firstLastQuery.LastPageSize < 1))
? firstLastQuery.FirstPageSize
: int.MaxValue;
Index targetIndexInfo = indexTypeMapping.IndexCollection[firstLastQuery.TargetIndexName];
#region Prepare Result
#region Extract index and apply criteria
CacheIndexInternal targetIndex = IndexServerUtils.GetCacheIndexInternal(storeContext,
messageContext.TypeId,
messageContext.PrimaryId,
firstLastQuery.IndexId,
targetIndexInfo.ExtendedIdSuffix,
firstLastQuery.TargetIndexName,
maxItemsPerIndex,
firstLastQuery.Filter,
true,
null,
false,
false,
targetIndexInfo.PrimarySortInfo,
targetIndexInfo.LocalIdentityTagList,
targetIndexInfo.StringHashCodeDictionary,
null);
#endregion
if (targetIndex != null)
{
indexSize = targetIndex.OutDeserializationContext.TotalCount;
virtualCount = targetIndex.VirtualCount;
indexExists = true;
// update perf counters
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsInIndexPerFirstLastQuery,
messageContext.TypeId,
indexSize);
PerformanceCounters.Instance.SetCounterValue(
PerformanceCounterEnum.NumOfItemsReadPerFirstLastQuery,
messageContext.TypeId,
targetIndex.OutDeserializationContext.ReadItemCount);
#region Populate resultLists
if (firstLastQuery.FirstPageSize + firstLastQuery.LastPageSize <= targetIndex.Count)
{
firstPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, firstLastQuery.FirstPageSize);
lastPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex,
targetIndex.Count - firstLastQuery.LastPageSize + 1,
firstLastQuery.LastPageSize);
}
else
{
//Populate everything in firstPageResultItemList
firstPageResultItemList = CacheIndexInternalAdapter.GetResultItemList(targetIndex, 1, targetIndex.Count);
}
#endregion
#region Get data
if (!firstLastQuery.ExcludeData)
{
//First Page
if (firstPageResultItemList != null)
{
DataTierUtil.GetData(firstPageResultItemList,
storeContext,
messageContext,
indexTypeMapping.FullDataIdFieldList,
firstLastQuery.FullDataIdInfo);
}
//Last Page
if (lastPageResultItemList != null)
{
DataTierUtil.GetData(lastPageResultItemList,
storeContext,
messageContext,
indexTypeMapping.FullDataIdFieldList,
firstLastQuery.FullDataIdInfo);
}
}
#endregion
#region Get metadata
if (firstLastQuery.GetMetadata)
{
metadata = IndexServerUtils.GetQueryMetadata(new List<CacheIndexInternal>(1) { targetIndex },
indexTypeMapping,
messageContext.TypeId,
messageContext.PrimaryId,
firstLastQuery.IndexId,
storeContext);
}
#endregion
}
#endregion
firstLastQueryResult = new FirstLastQueryResult(indexExists, indexSize, metadata, firstPageResultItemList, lastPageResultItemList, virtualCount, null);
}
catch (Exception ex)
{
firstLastQueryResult = new FirstLastQueryResult(false, -1, null, null, null, -1, ex.Message);
LoggingUtil.Log.ErrorFormat("TypeID {0} -- Error processing FirstLastQuery : {1}", messageContext.TypeId, ex);
}
return firstLastQueryResult;
}
