public IndexOperation(IndexOperationType action, string keyField, string keyValue)
{
Action = action;
Properties = new Dictionary <string, object> {
{ keyField, keyValue }
};
}
public void ReturnsEmptyOperationForTooLargeSkip(int skip, IndexOperationType expected)
{
AutocompleteRequest.Skip = skip;
V2SearchRequest.Skip = skip;
V3SearchRequest.Skip = skip;
var actual = Build();
Assert.Equal(expected, actual.Type);
}
public async Task <bool> DirectApplyIndexUpdateBatch(Immutable <IDictionary <IIndexableGrain, IList <IMemberUpdate> > > iUpdates, bool isUnique, IndexMetaData idxMetaData, SiloAddress siloAddress = null)
{
logger.Trace($"Started calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {isUnique}, siloAddress = {siloAddress}," +
$" iUpdates = {MemberUpdate.UpdatesToString(iUpdates.Value)}");
IDictionary <IIndexableGrain, IList <IMemberUpdate> > updates = iUpdates.Value;
IDictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > > bucketUpdates = new Dictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > >();
foreach (var kv in updates)
{
IIndexableGrain g = kv.Key;
IList <IMemberUpdate> gUpdates = kv.Value;
foreach (IMemberUpdate update in gUpdates)
{
IndexOperationType opType = update.OperationType;
if (opType == IndexOperationType.Update)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
if (befImgHash == aftImgHash)
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
else
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Delete));
AddUpdateToBucket(bucketUpdates, g, aftImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Insert));
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
AddUpdateToBucket(bucketUpdates, g, aftImgHash, update);
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
}
}
var results = await Task.WhenAll(bucketUpdates.Select(kv =>
{
BucketT bucket = GetGrain(IndexUtils.GetIndexGrainPrimaryKey(typeof(V), this._indexName) + "_" + kv.Key);
return(bucket.DirectApplyIndexUpdateBatch(kv.Value.AsImmutable(), isUnique, idxMetaData, siloAddress));
}));
logger.Trace($"Finished calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {isUnique}, siloAddress = {siloAddress}," +
$" iUpdates = {MemberUpdate.UpdatesToString(iUpdates.Value)}, results = '{string.Join(", ", results)}'");
return(true);
}
private IndexOperation(
IndexOperationType type,
string documentKey,
string searchText,
SearchParameters searchParameters)
{
Type = type;
DocumentKey = documentKey;
SearchText = searchText;
SearchParameters = searchParameters;
}
/// <summary>
/// Combines two OperationTypes
/// </summary>
/// <param name="thisOp">OperationType1</param>
/// <param name="otherOp">OperationType2</param>
/// <returns></returns>
public static IndexOperationType CombineWith(this IndexOperationType thisOp, IndexOperationType otherOp)
{
switch (thisOp)
{
case IndexOperationType.None:
return(otherOp);
case IndexOperationType.Insert:
switch (otherOp)
{
case IndexOperationType.Insert:
throw new Exception(string.Format("Two subsequent Insert operations are not allowed."));
case IndexOperationType.Update:
return(IndexOperationType.Insert);
case IndexOperationType.Delete:
return(IndexOperationType.None);
default: //case IndexOperationType.None
return(thisOp);
}
case IndexOperationType.Update:
switch (otherOp)
{
case IndexOperationType.Insert:
throw new Exception(string.Format("An Insert operation after an Update operation is not allowed."));
case IndexOperationType.Delete:
return(otherOp); //i.e., IndexOperationType.Delete
default: //case IndexOperationType.None or IndexOperationType.Update
return(thisOp);
}
case IndexOperationType.Delete:
switch (otherOp)
{
case IndexOperationType.Insert:
return(IndexOperationType.Update);
case IndexOperationType.Update:
throw new Exception(string.Format("An Update operation after a Delete operation is not allowed."));
default: //case IndexOperationType.None or IndexOperationType.Delete
return(thisOp);
}
default:
throw new Exception(string.Format("Operation type {0} is not valid", thisOp));
}
}
public MemberUpdate(object befImg, object aftImg, IndexOperationType opType)
{
this._opType = opType;
if (opType == IndexOperationType.Update || opType == IndexOperationType.Delete)
{
this._befImg = befImg;
}
if (opType == IndexOperationType.Update || opType == IndexOperationType.Insert)
{
this._aftImg = aftImg;
}
}
public IndexOperationType GetOperationType()
{
IndexOperationType op = _update.GetOperationType();
switch (op)
{
case IndexOperationType.Delete: return(IndexOperationType.Insert);
case IndexOperationType.Insert: return(IndexOperationType.Delete);
default: return(op);
}
}
public async Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
IMemberUpdate update = iUpdate.Value;
IndexOperationType opType = update.GetOperationType();
if (opType == IndexOperationType.Update)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
int aftImgHash = update.GetAfterImage().GetHashCode();
BucketT befImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
if (befImgHash == aftImgHash)
{
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else
{
BucketT aftImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
var befTask = befImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverridenOperation(iUpdate.Value, IndexOperationType.Delete).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
var aftTask = aftImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverridenOperation(iUpdate.Value, IndexOperationType.Insert).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
bool[] results = await Task.WhenAll(befTask, aftTask);
return(results[0] && results[1]);
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = update.GetAfterImage().GetHashCode();
BucketT aftImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + aftImgHash
);
return(await aftImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
BucketT befImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
return(true);
}
public async Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
IMemberUpdate update = iUpdate.Value;
IndexOperationType opType = update.OperationType;
if (opType == IndexOperationType.Update)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
BucketT befImgBucket = this.GetBucketGrain(befImgHash);
if (befImgHash == aftImgHash)
{
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
BucketT aftImgBucket = this.GetBucketGrain(aftImgHash);
var befTask = befImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverriddenOperation(iUpdate.Value, IndexOperationType.Delete).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
var aftTask = aftImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverriddenOperation(iUpdate.Value, IndexOperationType.Insert).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
bool[] results = await Task.WhenAll(befTask, aftTask);
return(results[0] && results[1]);
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
BucketT aftImgBucket = this.GetBucketGrain(aftImgHash);
return(await aftImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
BucketT befImgBucket = this.GetBucketGrain(befImgHash);
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
return(true);
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public IndexOperation(ValueSet valueSet, IndexOperationType operation)
{
ValueSet = valueSet;
Operation = operation;
}
public IndexOperation(IndexOperationType action, string keyField, string keyValue)
{
Action = action;
Properties = new Dictionary<string, object> {{keyField, keyValue}};
}
public MemberUpdateOverriddenOperation(IMemberUpdate update, IndexOperationType opType)
{
this._update = update;
this.OperationType = opType;
}
/// <summary>
/// This method contains the common functionality for updating
/// hash-index bucket.
/// </summary>
/// <typeparam name="K">key type</typeparam>
/// <typeparam name="V">value type</typeparam>
/// <param name="updatedGrain">the updated grain that is being indexed</param>
/// <param name="update">the update information</param>
/// <param name="opType">the update operation type, which might be different
/// from the update operation type inside the update parameter</param>
/// <param name="State">the index bucket to be updated</param>
/// <param name="isUniqueIndex">a flag to indicate whether the
/// hash-index has a uniqueness constraint</param>
/// <param name="befImg">output parameter: the before-image</param>
/// <param name="befEntry">output parameter: the index entry containing the before-image</param>
/// <param name="fixIndexUnavailableOnDelete">output parameter: this variable determines whether
/// index was still unavailable when we received a delete operation</param>
internal static bool UpdateBucket <K, V>(V updatedGrain, IMemberUpdate update, HashIndexBucketState <K, V> State, bool isUniqueIndex, IndexMetaData idxMetaData, out K befImg, out HashIndexSingleBucketEntry <V> befEntry, out bool fixIndexUnavailableOnDelete) where V : IIndexableGrain
{
fixIndexUnavailableOnDelete = false;
befImg = default(K);
befEntry = null;
bool isTentativeUpdate = isUniqueIndex && (update is MemberUpdateTentative);
IndexOperationType opType = update.GetOperationType();
HashIndexSingleBucketEntry <V> aftEntry;
if (opType == IndexOperationType.Update)
{
befImg = (K)update.GetBeforeImage();
K aftImg = (K)update.GetAfterImage();
if (State.IndexMap.TryGetValue(befImg, out befEntry) && befEntry.Values.Contains(updatedGrain))
{ //Delete and Insert
if (State.IndexMap.TryGetValue(aftImg, out aftEntry))
{
if (aftEntry.Values.Contains(updatedGrain))
{
if (isTentativeUpdate)
{
aftEntry.setTentativeInsert();
}
else
{
aftEntry.clearTentativeFlag();
befEntry.Remove(updatedGrain, isTentativeUpdate, isUniqueIndex);
}
}
else
{
if (isUniqueIndex && aftEntry.Values.Count > 0)
{
throw new UniquenessConstraintViolatedException(string.Format("The uniqueness property of index is violated after an update operation for before-image = {0}, after-image = {1} and grain = {2}", befImg, aftImg, updatedGrain.GetPrimaryKey()));
}
befEntry.Remove(updatedGrain, isTentativeUpdate, isUniqueIndex);
aftEntry.Add(updatedGrain, isTentativeUpdate, isUniqueIndex);
}
}
else
{
aftEntry = new HashIndexSingleBucketEntry <V>();
befEntry.Remove(updatedGrain, isTentativeUpdate, isUniqueIndex);
aftEntry.Add(updatedGrain, isTentativeUpdate, isUniqueIndex);
State.IndexMap.Add(aftImg, aftEntry);
}
}
else
{
if (idxMetaData.IsChainedBuckets())
{
return(false); //not found in this bucket
}
//Insert
if (State.IndexMap.TryGetValue(aftImg, out aftEntry))
{
if (!aftEntry.Values.Contains(updatedGrain))
{
if (isUniqueIndex && aftEntry.Values.Count > 0)
{
throw new UniquenessConstraintViolatedException(string.Format("The uniqueness property of index is violated after an update operation for (not found before-image = {0}), after-image = {1} and grain = {2}", befImg, aftImg, updatedGrain.GetPrimaryKey()));
}
aftEntry.Add(updatedGrain, isTentativeUpdate, isUniqueIndex);
}
else if (isTentativeUpdate)
{
aftEntry.setTentativeInsert();
}
else
{
aftEntry.clearTentativeFlag();
}
}
else
{
if (idxMetaData.IsCreatingANewBucketNecessary(State.IndexMap.Count()))
{
return(false);
}
aftEntry = new HashIndexSingleBucketEntry <V>();
aftEntry.Add(updatedGrain, isTentativeUpdate, isUniqueIndex);
State.IndexMap.Add(aftImg, aftEntry);
}
}
}
else if (opType == IndexOperationType.Insert)
{ // Insert
K aftImg = (K)update.GetAfterImage();
if (State.IndexMap.TryGetValue(aftImg, out aftEntry))
{
if (!aftEntry.Values.Contains(updatedGrain))
{
if (isUniqueIndex && aftEntry.Values.Count > 0)
{
throw new UniquenessConstraintViolatedException(string.Format("The uniqueness property of index is violated after an insert operation for after-image = {0} and grain = {1}", aftImg, updatedGrain.GetPrimaryKey()));
}
aftEntry.Add(updatedGrain, isTentativeUpdate, isUniqueIndex);
}
else if (isTentativeUpdate)
{
aftEntry.setTentativeInsert();
}
else
{
aftEntry.clearTentativeFlag();
}
}
else
{
if (idxMetaData.IsCreatingANewBucketNecessary(State.IndexMap.Count()))
{
return(false); //the bucket is full
}
aftEntry = new HashIndexSingleBucketEntry <V>();
aftEntry.Add(updatedGrain, isTentativeUpdate, isUniqueIndex);
State.IndexMap.Add(aftImg, aftEntry);
}
}
else if (opType == IndexOperationType.Delete)
{ // Delete
befImg = (K)update.GetBeforeImage();
if (State.IndexMap.TryGetValue(befImg, out befEntry) && befEntry.Values.Contains(updatedGrain))
{
befEntry.Remove(updatedGrain, isTentativeUpdate, isUniqueIndex);
if (State.IndexStatus != IndexStatus.Available)
{
fixIndexUnavailableOnDelete = true;
}
}
else if (idxMetaData.IsChainedBuckets())
{
return(false); //not found in this bucket
}
}
return(true);
}
public MemberUpdateOverridenOperation(IMemberUpdate update, IndexOperationType opType)
{
_update = update;
_opType = opType;
}
/// <summary>
/// 原子操作索引
/// </summary>
/// <param name="collectionName">操作的集合（solr中的集体名称）</param>
/// <param name="uniqueKeyName">主键名称（solr中的主键）</param>
/// <param name="uniqueKeyValue">主键值（solr中的主键值）</param>
/// <param name="setItem">需要操作的集体（key为字段名称，value为值）</param>
/// <param name="operationType">操作类型</param>
/// <returns>操作结果</returns>
public IndexOperationResponse AtomOperateIndex(string collectionName, string uniqueKeyName, object uniqueKeyValue, Dictionary <string, object> setItem, IndexOperationType operationType = IndexOperationType.Update)
{
var docs = new List <SolrInputDocument>();
var doc = new SolrInputDocument();
try
{
doc.Add(uniqueKeyName, new SolrInputField(uniqueKeyName, uniqueKeyValue));
foreach (var item in setItem)
{
var setOper = new Hashtable();
switch (operationType)
{
case IndexOperationType.Add:
setOper.Add("add", item.Value);
break;
case IndexOperationType.Update:
setOper.Add("set", item.Value);
break;
default:
setOper.Add("set", item.Value);
break;
}
doc.Add(item.Key, new SolrInputField(item.Key, setOper));
docs.Add(doc);
}
collectionName = string.IsNullOrEmpty(collectionName) ? searchConfig.Collection : collectionName;
var result = updateOperations.Update(collectionName, "/update", new UpdateOptions()
{
OptimizeOptions = optimizeOptions, Docs = docs
});
var header = binaryResponseHeaderParser.Parse(result);
return(new IndexOperationResponse
{
IsSuccess = header.Status == 0,
Message = string.Empty,
QTime = header.QTime,
Status = header.Status
});
}
catch (Exception ex)
{
return(new IndexOperationResponse
{
IsSuccess = false,
Message = ex.Message,
QTime = -1,
Status = -1
});
}
}
public async Task <bool> DirectApplyIndexUpdateBatch(Immutable <IDictionary <IIndexableGrain, IList <IMemberUpdate> > > iUpdates, bool isUnique, IndexMetaData idxMetaData, SiloAddress siloAddress = null)
{
if (logger.IsVerbose)
{
logger.Verbose("Started calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {0}, siloAddress = {1}, iUpdates = {2}", isUnique, siloAddress, MemberUpdate.UpdatesToString(iUpdates.Value));
}
IDictionary <IIndexableGrain, IList <IMemberUpdate> > updates = iUpdates.Value;
IDictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > > bucketUpdates = new Dictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > >();
foreach (var kv in updates)
{
IIndexableGrain g = kv.Key;
IList <IMemberUpdate> gUpdates = kv.Value;
foreach (IMemberUpdate update in gUpdates)
{
IndexOperationType opType = update.GetOperationType();
if (opType == IndexOperationType.Update)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
int aftImgHash = update.GetAfterImage().GetHashCode();
if (befImgHash == aftImgHash)
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
else
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Delete));
AddUpdateToBucket(bucketUpdates, g, aftImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Insert));
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = update.GetAfterImage().GetHashCode();
AddUpdateToBucket(bucketUpdates, g, aftImgHash, update);
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
}
}
List <Task> updateTasks = new List <Task>();
int i = 0;
foreach (var kv in bucketUpdates)
{
BucketT bucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + kv.Key
);
updateTasks.Add(bucket.DirectApplyIndexUpdateBatch(kv.Value.AsImmutable(), isUnique, idxMetaData, siloAddress));
++i;
}
await Task.WhenAll(updateTasks);
if (logger.IsVerbose)
{
logger.Verbose("Finished calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {0}, siloAddress = {1}, iUpdates = {2}", isUnique, siloAddress, MemberUpdate.UpdatesToString(iUpdates.Value));
}
return(true);
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public IndexOperation(IndexItem item, IndexOperationType operation)
{
Item = item;
Operation = operation;
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public IndexOperation(IndexItem item, IndexOperationType operation)
{
Item = item;
Operation = operation;
}
