private Task ApplyIndexUpdatesEagerly(Type iGrainType,
IIndexableGrain updatedGrain,
IDictionary <string, IMemberUpdate> updates,
UpdateIndexType updateIndexTypes,
bool updateIndexesTentatively)
{
IList <Task <bool> > updateIndexTasks = new List <Task <bool> >();
foreach (KeyValuePair <string, IMemberUpdate> updt in updates.Where(updt => updt.Value.OperationType != IndexOperationType.None))
{
var idxInfo = this._grainIndexes[updt.Key];
if (updateIndexTypes.HasFlag(idxInfo.MetaData.IsUniqueIndex ? UpdateIndexType.Unique : UpdateIndexType.NonUnique))
{
// If the caller asks for the update to be tentative, then it will be wrapped inside a MemberUpdateTentative
IMemberUpdate updateToIndex = updateIndexesTentatively ? new MemberUpdateTentative(updt.Value) : updt.Value;
// The update task is added to the list of update tasks
updateIndexTasks.Add(idxInfo.IndexInterface.ApplyIndexUpdate(this.SiloIndexManager,
updatedGrain, updateToIndex.AsImmutable(), idxInfo.MetaData, BaseSiloAddress));
}
}
// At the end, because the index update should be eager, we wait for all index update tasks to finish
return(Task.WhenAll(updateIndexTasks));
}
private void ApplyIndexUpdatesLazilyWithoutWait(IDictionary <string, IMemberUpdate> updates,
IList <Type> iGrainTypes,
IIndexableGrain thisGrain,
Guid workflowID)
{
ApplyIndexUpdatesLazily(updates, iGrainTypes, thisGrain, workflowID).Ignore();
}
private Dictionary <string, IDictionary <IIndexableGrain, IList <IMemberUpdate> > > PopulateUpdatesToIndexes(
IndexWorkflowRecordNode currentWorkflow, Dictionary <IIndexableGrain, HashSet <Guid> > grainsToActiveWorkflows)
{
var updatesToIndexes = new Dictionary <string, IDictionary <IIndexableGrain, IList <IMemberUpdate> > >();
bool faultTolerant = IsFaultTolerant;
for (; !currentWorkflow.IsPunctuation; currentWorkflow = currentWorkflow.Next)
{
IndexWorkflowRecord workflowRec = currentWorkflow.WorkflowRecord;
IIndexableGrain g = workflowRec.Grain;
bool existsInActiveWorkflows = faultTolerant && grainsToActiveWorkflows.TryGetValue(g, out HashSet <Guid> activeWorkflowRecs) &&
activeWorkflowRecs.Contains(workflowRec.WorkflowId);
foreach (var(indexName, updt) in currentWorkflow.WorkflowRecord.MemberUpdates.Where(kvp => kvp.Value.OperationType != IndexOperationType.None))
{
var updatesByGrain = updatesToIndexes.GetOrAdd(indexName, () => new Dictionary <IIndexableGrain, IList <IMemberUpdate> >());
var updatesForGrain = updatesByGrain.GetOrAdd(g, () => new List <IMemberUpdate>());
if (!faultTolerant || existsInActiveWorkflows)
{
updatesForGrain.Add(updt);
}
else if (GrainIndexes[indexName].MetaData.IsUniqueIndex)
{
// If the workflow record does not exist in the set of active workflows and the index is fault-tolerant,
// enqueue a reversal (undo) to any possible remaining tentative updates to unique indexes.
updatesForGrain.Add(new MemberUpdateReverseTentative(updt));
}
}
}
return(updatesToIndexes);
}
private async Task DirectApplyIndexUpdateNonPersistent(IIndexableGrain g, IMemberUpdate updt, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
//this variable determines whether index was still unavailable
//when we received a delete operation
bool fixIndexUnavailableOnDelete = false;
//the target grain that is updated
V updatedGrain = g.AsReference <V>(GrainFactory);
K befImg;
HashIndexSingleBucketEntry <V> befEntry;
//Updates the index bucket synchronously
//(note that no other thread can run concurrently
//before we reach an await operation, so no concurrency
//control mechanism (e.g., locking) is required)
if (!HashIndexBucketUtils.UpdateBucket(updatedGrain, updt, State, isUniqueIndex, idxMetaData, out befImg, out befEntry, out fixIndexUnavailableOnDelete))
{
await(await GetNextBucketAndPersist()).DirectApplyIndexUpdate(g, updt.AsImmutable(), isUniqueIndex, idxMetaData, siloAddress);
}
//if the index was still unavailable
//when we received a delete operation
//if (fixIndexUnavailableOnDelete)
//{
// //create tombstone
//}
}
private Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, IMemberUpdate updt, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
V updatedGrain = g;
HashIndexBucketUtils.UpdateBucket(updatedGrain, updt, state, isUniqueIndex, idxMetaData);
return(Task.FromResult(true));
}
private async Task DirectApplyIndexUpdates(IIndexableGrain g, IList <IMemberUpdate> updates, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
foreach (IMemberUpdate updt in updates)
{
await DirectApplyIndexUpdate(g, updt, isUniqueIndex, idxMetaData, siloAddress);
}
}
private async Task <Dictionary <IIndexableGrain, HashSet <Guid> > > GetActiveWorkflowsListsFromGrains(IndexWorkflowRecordNode currentWorkflow)
{
var result = new Dictionary <IIndexableGrain, HashSet <Guid> >();
var grains = new List <IIndexableGrain>();
var activeWorkflowsSetsTasks = new List <Task <Immutable <HashSet <Guid> > > >();
while (!currentWorkflow.IsPunctuation())
{
IIndexableGrain g = currentWorkflow.WorkflowRecord.Grain;
foreach (var updates in currentWorkflow.WorkflowRecord.MemberUpdates)
{
IMemberUpdate updt = updates.Value;
if (updt.GetOperationType() != IndexOperationType.None && !result.ContainsKey(g))
{
result.Add(g, EMPTY_HASHSET);
grains.Add(g);
activeWorkflowsSetsTasks.Add(g.AsReference <IIndexableGrain>(InsideRuntimeClient.Current.ConcreteGrainFactory, _iGrainType).GetActiveWorkflowIdsList());
}
}
currentWorkflow = currentWorkflow.Next;
}
if (activeWorkflowsSetsTasks.Count() > 0)
{
Immutable <HashSet <Guid> >[] activeWorkflowsSets = await Task.WhenAll(activeWorkflowsSetsTasks);
for (int i = 0; i < activeWorkflowsSets.Length; ++i)
{
result[grains[i]] = activeWorkflowsSets[i].Value;
}
}
return(result);
}
private protected async Task PreActivate(Grain grain, Func <Task <IndexedGrainStateWrapper <TGrainState> > > readGrainStateFunc, Func <Task> writeGrainStateFunc)
{
if (this.grain != null) // Already called
{
return;
}
this.grain = grain;
this.iIndexableGrain = this.grain.AsReference <IIndexableGrain>(this.SiloIndexManager);
this.readGrainStateFunc = readGrainStateFunc;
this.writeGrainStateFunc = writeGrainStateFunc;
await this.ReadAsync();
if (!GrainIndexes.CreateInstance(this.SiloIndexManager.IndexRegistry, this.grain.GetType(), out this._grainIndexes) ||
!this._grainIndexes.HasAnyIndexes)
{
throw new InvalidOperationException("IndexedState should not be used for a Grain class with no indexes");
}
if (!this.wrappedState.AreNullValuesInitialized)
{
IndexUtils.SetNullValues(this.wrappedState.UserState, this._grainIndexes.PropertyNullValues);
this.wrappedState.AreNullValuesInitialized = true;
}
this._hasAnyUniqueIndex = this._grainIndexes.HasAnyUniqueIndex;
this._grainIndexes.AddMissingBeforeImages(this.wrappedState.UserState);
}
private async Task <Dictionary <IIndexableGrain, HashSet <Guid> > > FtGetActiveWorkflowSetsFromGrains(IndexWorkflowRecordNode currentWorkflow)
{
var activeWorkflowSetTasksByGrain = new Dictionary <IIndexableGrain, Task <Immutable <HashSet <Guid> > > >();
var currentWorkflowIds = new HashSet <Guid>();
for (; !currentWorkflow.IsPunctuation; currentWorkflow = currentWorkflow.Next)
{
var record = currentWorkflow.WorkflowRecord;
currentWorkflowIds.Add(record.WorkflowId);
IIndexableGrain g = record.Grain;
if (!activeWorkflowSetTasksByGrain.ContainsKey(g) && record.MemberUpdates.Where(ups => ups.Value.OperationType != IndexOperationType.None).Any())
{
activeWorkflowSetTasksByGrain[g] = g.AsReference <IIndexableGrain>(this._siloIndexManager, this._grainInterfaceType).GetActiveWorkflowIdsSet();
}
}
if (activeWorkflowSetTasksByGrain.Count > 0)
{
await Task.WhenAll(activeWorkflowSetTasksByGrain.Values);
// Intersect so we do not include workflowIds that are not in our work queue.
return(activeWorkflowSetTasksByGrain.ToDictionary(kvp => kvp.Key,
kvp => new HashSet <Guid>(kvp.Value.Result.Value.Intersect(currentWorkflowIds))));
}
return(new Dictionary <IIndexableGrain, HashSet <Guid> >());
}
/// <summary>
/// This method applies a given update to the current index.
/// </summary>
/// <param name="updatedGrain">the grain that issued the update</param>
/// <param name="iUpdate">contains the data for the update</param>
/// <param name="isUnique">whether this is a unique index that we are updating</param>
/// <param name="op">the actual type of the operation, which override the operation-type in iUpdate</param>
/// <returns>true, if the index update was successful, otherwise false</returns>
public async Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
await DirectApplyIndexUpdateNonPersistent(g, iUpdate.Value, isUniqueIndex, idxMetaData, siloAddress);
await PersistIndex();
return(true);
}
private Task DirectApplyIndexUpdatesNonPersistent(IIndexableGrain g, IList <IMemberUpdate> updates, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
Task[] updateTasks = new Task[updates.Count()];
int i = 0;
foreach (IMemberUpdate updt in updates)
{
updateTasks[i++] = DirectApplyIndexUpdateNonPersistent(g, updt, isUniqueIndex, idxMetaData, siloAddress);
}
return(Task.WhenAll(updateTasks));
}
public async Task <bool> DirectApplyIndexUpdateBatch(Immutable <IDictionary <IIndexableGrain, IList <IMemberUpdate> > > iUpdates, bool isUnique, IndexMetaData idxMetaData, SiloAddress siloAddress = null)
{
logger.Trace($"Started calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {isUnique}, siloAddress = {siloAddress}," +
$" iUpdates = {MemberUpdate.UpdatesToString(iUpdates.Value)}");
IDictionary <IIndexableGrain, IList <IMemberUpdate> > updates = iUpdates.Value;
IDictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > > bucketUpdates = new Dictionary <int, IDictionary <IIndexableGrain, IList <IMemberUpdate> > >();
foreach (var kv in updates)
{
IIndexableGrain g = kv.Key;
IList <IMemberUpdate> gUpdates = kv.Value;
foreach (IMemberUpdate update in gUpdates)
{
IndexOperationType opType = update.OperationType;
if (opType == IndexOperationType.Update)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
if (befImgHash == aftImgHash)
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
else
{
AddUpdateToBucket(bucketUpdates, g, befImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Delete));
AddUpdateToBucket(bucketUpdates, g, aftImgHash, new MemberUpdateOverridenOperation(update, IndexOperationType.Insert));
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
AddUpdateToBucket(bucketUpdates, g, aftImgHash, update);
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
AddUpdateToBucket(bucketUpdates, g, befImgHash, update);
}
}
}
var results = await Task.WhenAll(bucketUpdates.Select(kv =>
{
BucketT bucket = GetGrain(IndexUtils.GetIndexGrainPrimaryKey(typeof(V), this._indexName) + "_" + kv.Key);
return(bucket.DirectApplyIndexUpdateBatch(kv.Value.AsImmutable(), isUnique, idxMetaData, siloAddress));
}));
logger.Trace($"Finished calling DirectApplyIndexUpdateBatch with the following parameters: isUnique = {isUnique}, siloAddress = {siloAddress}," +
$" iUpdates = {MemberUpdate.UpdatesToString(iUpdates.Value)}, results = '{string.Join(", ", results)}'");
return(true);
}
protected Task ApplyIndexUpdatesLazily(IDictionary <string, IMemberUpdate> updates,
IList <Type> iGrainTypes,
IIndexableGrain thisGrain,
Guid workflowID)
{
Task addToQueue(Type iGrainType) => GetWorkflowQueue(iGrainType).AddToQueue(new IndexWorkflowRecord(workflowID, thisGrain, updates).AsImmutable());
return(iGrainTypes.Count() == 1
? addToQueue(iGrainTypes[0])
: Task.WhenAll(iGrainTypes.Select(iGrainType => addToQueue(iGrainType))));
}
protected async Task ApplyIndexUpdatesEagerly(IList <Type> iGrainTypes,
IIndexableGrain updatedGrain,
IDictionary <string, IMemberUpdate> updates,
UpdateIndexType updateIndexTypes,
bool updateIndexesTentatively = false)
{
Task applyUpdate(Type iGrainType) => ApplyIndexUpdatesEagerly(iGrainType, updatedGrain, updates, updateIndexTypes, updateIndexesTentatively);
await(iGrainTypes.Count() == 1
? applyUpdate(iGrainTypes[0])
: Task.WhenAll(iGrainTypes.Select(iGrainType => applyUpdate(iGrainType))));
}
/// <summary>
/// An extension method to intercept the calls to DirectApplyIndexUpdate
/// on an Index
/// </summary>
internal static Task <bool> ApplyIndexUpdate(this IndexInterface index, IIndexableGrain updatedGrain, Immutable <IMemberUpdate> update, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress = null)
{
if (index is ActiveHashIndexPartitionedPerSilo)
{
ActiveHashIndexPartitionedPerSiloBucket bucketInCurrentSilo = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <ActiveHashIndexPartitionedPerSiloBucket>(
GetAHashIndexPartitionedPerSiloGrainID(IndexUtils.GetIndexNameFromIndexGrain((IAddressable )index), index.GetType().GetGenericArguments()[1]),
siloAddress
);
return(bucketInCurrentSilo.DirectApplyIndexUpdate(updatedGrain, update, isUniqueIndex, idxMetaData /*, siloAddress*/));
}
return(index.DirectApplyIndexUpdate(updatedGrain, update, isUniqueIndex, idxMetaData, siloAddress));
}
private void PopulateUpdatesToIndexes(IndexWorkflowRecordNode currentWorkflow, Dictionary <string, IDictionary <IIndexableGrain, IList <IMemberUpdate> > > updatesToIndexes, Dictionary <IIndexableGrain, HashSet <Guid> > grainsToActiveWorkflows)
{
bool faultTolerant = IsFaultTolerant;
while (!currentWorkflow.IsPunctuation())
{
IndexWorkflowRecord workflowRec = currentWorkflow.WorkflowRecord;
IIndexableGrain g = workflowRec.Grain;
bool existsInActiveWorkflows = false;
if (faultTolerant)
{
HashSet <Guid> activeWorkflowRecs = null;
if (grainsToActiveWorkflows.TryGetValue(g, out activeWorkflowRecs))
{
if (activeWorkflowRecs.Contains(workflowRec.WorkflowId))
{
existsInActiveWorkflows = true;
}
}
}
foreach (var updates in currentWorkflow.WorkflowRecord.MemberUpdates)
{
IMemberUpdate updt = updates.Value;
if (updt.GetOperationType() != IndexOperationType.None)
{
string index = updates.Key;
var updatesToIndex = updatesToIndexes[index];
IList <IMemberUpdate> updatesList;
if (!updatesToIndex.TryGetValue(g, out updatesList))
{
updatesList = new List <IMemberUpdate>();
updatesToIndex.Add(g, updatesList);
}
if (!faultTolerant || existsInActiveWorkflows)
{
updatesList.Add(updt);
}
//if the workflow record does not exist in the list of active work-flows
//and the index is fault-tolerant, we should make sure that tentative updates
//to unique indexes are undone
else if (((IndexMetaData)Indexes[index].Item2).IsUniqueIndex())
{
//reverse a possible remaining tentative record from the index
updatesList.Add(new MemberUpdateReverseTentative(updt));
}
}
}
currentWorkflow = currentWorkflow.Next;
}
}
/// <summary>
/// An extension method to intercept the calls to DirectApplyIndexUpdate on an Index,
/// so that for a PerSilo index, it can obtain the GrainService of that index on the silo of the indexed grain.
/// </summary>
internal static Task <bool> ApplyIndexUpdate(this IIndexInterface index, SiloIndexManager siloIndexManager,
IIndexableGrain updatedGrain, Immutable <IMemberUpdate> update,
IndexMetaData idxMetaData, SiloAddress siloAddress = null)
{
if (index is IActiveHashIndexPartitionedPerSilo)
{
var bucketInCurrentSilo = siloIndexManager.GetGrainService <IActiveHashIndexPartitionedPerSiloBucket>(
GetAHashIndexPartitionedPerSiloGrainReference(siloIndexManager,
IndexUtils.GetIndexNameFromIndexGrain((IAddressable)index), index.GetType().GetGenericArguments()[1],
siloAddress
));
return(bucketInCurrentSilo.DirectApplyIndexUpdate(updatedGrain, update, idxMetaData.IsUniqueIndex, idxMetaData /*, siloAddress*/));
}
return(index.DirectApplyIndexUpdate(updatedGrain, update, idxMetaData.IsUniqueIndex, idxMetaData, siloAddress));
}
private Task Initialize(Grain grain)
{
if (this.grain == null) // If not already called
{
this.grain = grain;
this.iIndexableGrain = this.grain.AsReference <IIndexableGrain>(this.SiloIndexManager);
if (!GrainIndexes.CreateInstance(this.SiloIndexManager.IndexRegistry, this.grain.GetType(), out this._grainIndexes) ||
!this._grainIndexes.HasAnyIndexes)
{
throw new InvalidOperationException("IndexedState should not be used for a Grain class with no indexes");
}
this._hasAnyUniqueIndex = this._grainIndexes.HasAnyUniqueIndex;
}
return(Task.CompletedTask);
}
public async Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
IMemberUpdate update = iUpdate.Value;
IndexOperationType opType = update.GetOperationType();
if (opType == IndexOperationType.Update)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
int aftImgHash = update.GetAfterImage().GetHashCode();
BucketT befImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
if (befImgHash == aftImgHash)
{
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else
{
BucketT aftImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
var befTask = befImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverridenOperation(iUpdate.Value, IndexOperationType.Delete).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
var aftTask = aftImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverridenOperation(iUpdate.Value, IndexOperationType.Insert).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
bool[] results = await Task.WhenAll(befTask, aftTask);
return(results[0] && results[1]);
}
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = update.GetAfterImage().GetHashCode();
BucketT aftImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + aftImgHash
);
return(await aftImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = update.GetBeforeImage().GetHashCode();
BucketT befImgBucket = InsideRuntimeClient.Current.InternalGrainFactory.GetGrain <BucketT>(
IndexUtils.GetIndexGrainID(typeof(V), _indexName) + "_" + befImgHash
);
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
return(true);
}
private async Task DirectApplyIndexUpdateNonPersistent(IIndexableGrain g, IMemberUpdate updt, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
// The target grain that is updated
V updatedGrain = g.AsReference <V>(this.SiloIndexManager);
// Updates the index bucket synchronously (note that no other thread can run concurrently before we reach an await operation,
// when execution is yielded back to the Orleans scheduler, so no concurrency control mechanism (e.g., locking) is required).
// 'fixIndexUnavailableOnDelete' indicates whether the index was still unavailable when we received a delete operation.
if (!HashIndexBucketUtils.UpdateBucketState(updatedGrain, updt, this.State, isUniqueIndex, idxMetaData, out K befImg, out HashIndexSingleBucketEntry <V> befEntry, out bool fixIndexUnavailableOnDelete))
{
await(await GetNextBucketAndPersist()).DirectApplyIndexUpdate(g, updt.AsImmutable(), isUniqueIndex, idxMetaData, siloAddress);
}
// TODO if the index was still unavailable when we received a delete operation
//if (fixIndexUnavailableOnDelete)
//{
// //create tombstone
//}
}
private Task ApplyIndexUpdatesEagerly(Type iGrainType,
IIndexableGrain updatedGrain,
IDictionary <string, IMemberUpdate> updates,
bool onlyUpdateUniqueIndexes,
bool onlyUpdateNonUniqueIndexes,
bool updateIndexesTentatively)
{
IList <Task <bool> > updateIndexTasks = new List <Task <bool> >();
foreach (KeyValuePair <string, IMemberUpdate> updt in updates)
{
//if the update is not a no-operation
if (updt.Value.GetOperationType() != IndexOperationType.None)
{
var idxInfo = _iUpdateGens[updt.Key];
var isUniqueIndex = ((IndexMetaData)idxInfo.Item2).IsUniqueIndex();
//the actual update happens if either the corresponding index is not a unique index
//and the caller asks for only updating non-unique indexes, or the corresponding
//index is a unique index and the caller asks for only updating unqiue indexes.
if ((onlyUpdateNonUniqueIndexes && !isUniqueIndex) || (onlyUpdateUniqueIndexes && isUniqueIndex))
{
IMemberUpdate updateToIndex = updt.Value;
//if the caller asks for the update to be tentative, then
//it will be wrapped inside a MemberUpdateTentative
if (updateIndexesTentatively)
{
updateToIndex = new MemberUpdateTentative(updateToIndex);
}
//the update task is added to the list of update tasks
updateIndexTasks.Add(((IndexInterface)idxInfo.Item1).ApplyIndexUpdate(updatedGrain, updateToIndex.AsImmutable(), isUniqueIndex, (IndexMetaData)idxInfo.Item2, RuntimeAddress));
}
}
}
//at the end, because the index update should be eager, we wait for
//all index update tasks to finish
return(Task.WhenAll(updateIndexTasks));
}
protected async Task ApplyIndexUpdatesEagerly(IList <Type> iGrainTypes,
IIndexableGrain updatedGrain,
IDictionary <string, IMemberUpdate> updates,
bool onlyUpdateUniqueIndexes,
bool onlyUpdateNonUniqueIndexes,
bool updateIndexesTentatively = false)
{
if (iGrainTypes.Count() == 1)
{
await ApplyIndexUpdatesEagerly(iGrainTypes[0], updatedGrain, updates, onlyUpdateUniqueIndexes, onlyUpdateNonUniqueIndexes, updateIndexesTentatively);
}
else
{
Task[] updateTasks = new Task[iGrainTypes.Count()];
int i = 0;
foreach (Type iGrainType in iGrainTypes)
{
updateTasks[i++] = ApplyIndexUpdatesEagerly(iGrainType, updatedGrain, updates, onlyUpdateUniqueIndexes, onlyUpdateNonUniqueIndexes, updateIndexesTentatively);
}
await Task.WhenAll(updateTasks);
}
}
protected Task ApplyIndexUpdatesLazily(IDictionary <string, IMemberUpdate> updates,
IList <Type> iGrainTypes,
IIndexableGrain thisGrain,
Guid workflowID)
{
if (iGrainTypes.Count() == 1)
{
IIndexWorkflowQueue workflowQ = GetWorkflowQueue(iGrainTypes[0]);
return(workflowQ.AddToQueue(new IndexWorkflowRecord(workflowID, thisGrain, updates).AsImmutable()));
}
else
{
Task[] tasks = new Task[iGrainTypes.Count];
int i = 0;
foreach (Type iGrainType in iGrainTypes)
{
tasks[i++] = GetWorkflowQueue(iGrainType).AddToQueue(
new IndexWorkflowRecord(workflowID, thisGrain, updates).AsImmutable()
);
}
return(Task.WhenAll(tasks));
}
}
private async Task <Dictionary <IIndexableGrain, HashSet <Guid> > > GetActiveWorkflowsListsFromGrains(IndexWorkflowRecordNode currentWorkflow)
{
var result = new Dictionary <IIndexableGrain, HashSet <Guid> >();
var grains = new List <IIndexableGrain>();
var activeWorkflowsSetsTasks = new List <Task <Immutable <HashSet <Guid> > > >();
var workflowIds = new HashSet <Guid>();
while (!currentWorkflow.IsPunctuation())
{
workflowIds.Add(currentWorkflow.WorkflowRecord.WorkflowId);
IIndexableGrain g = currentWorkflow.WorkflowRecord.Grain;
foreach (var updates in currentWorkflow.WorkflowRecord.MemberUpdates)
{
IMemberUpdate updt = updates.Value;
if (updt.OperationType != IndexOperationType.None && !result.ContainsKey(g))
{
result.Add(g, emptyHashset);
grains.Add(g);
activeWorkflowsSetsTasks.Add(g.AsReference <IIndexableGrain>(_siloIndexManager, _iGrainType).GetActiveWorkflowIdsList());
}
}
currentWorkflow = currentWorkflow.Next;
}
if (activeWorkflowsSetsTasks.Count() > 0)
{
Immutable <HashSet <Guid> >[] activeWorkflowsSets = await Task.WhenAll(activeWorkflowsSetsTasks);
for (int i = 0; i < activeWorkflowsSets.Length; ++i)
{
// Do not include workflowIds that are not in our work queue.
result[grains[i]] = new HashSet <Guid>(activeWorkflowsSets[i].Value.Intersect(workflowIds));
}
}
return(result);
}
public async Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress)
{
IMemberUpdate update = iUpdate.Value;
IndexOperationType opType = update.OperationType;
if (opType == IndexOperationType.Update)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
BucketT befImgBucket = this.GetBucketGrain(befImgHash);
if (befImgHash == aftImgHash)
{
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
BucketT aftImgBucket = this.GetBucketGrain(aftImgHash);
var befTask = befImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverriddenOperation(iUpdate.Value, IndexOperationType.Delete).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
var aftTask = aftImgBucket.DirectApplyIndexUpdate(g, new MemberUpdateOverriddenOperation(iUpdate.Value, IndexOperationType.Insert).AsImmutable <IMemberUpdate>(), isUniqueIndex, idxMetaData);
bool[] results = await Task.WhenAll(befTask, aftTask);
return(results[0] && results[1]);
}
else if (opType == IndexOperationType.Insert)
{
int aftImgHash = GetBucketIndexFromHashCode(update.GetAfterImage());
BucketT aftImgBucket = this.GetBucketGrain(aftImgHash);
return(await aftImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
else if (opType == IndexOperationType.Delete)
{
int befImgHash = GetBucketIndexFromHashCode(update.GetBeforeImage());
BucketT befImgBucket = this.GetBucketGrain(befImgHash);
return(await befImgBucket.DirectApplyIndexUpdate(g, iUpdate, isUniqueIndex, idxMetaData));
}
return(true);
}
/// <summary> /// DirectApplyIndexUpdate is not supported on ActiveHashIndexPartitionedPerSiloImpl, because it will be skipped /// via IndexExtensions.ApplyIndexUpdate which goes directly to the IActiveHashIndexPartitionedPerSiloBucket grain service /// </summary> public Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress) => throw new NotSupportedException();
public Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex, IndexMetaData idxMetaData, SiloAddress siloAddress) => DirectApplyIndexUpdate(g, iUpdate.Value, isUniqueIndex, idxMetaData, siloAddress);
/// <summary>
/// Applies a set of updates to the indexes defined on the grain
/// </summary>
/// <param name="updates">the dictionary of indexes to their corresponding updates</param>
/// <param name="updateIndexesEagerly">whether indexes should be
/// updated eagerly or lazily</param>
/// <param name="onlyUniqueIndexesWereUpdated">a flag to determine whether
/// only unique indexes were updated</param>
/// <param name="numberOfUniqueIndexesUpdated">determine the number of
/// updated unique indexes</param>
/// <param name="writeStateIfConstraintsAreNotViolated">whether writing back
/// the state to the storage should be done if no constraint is violated</param>
protected virtual async Task ApplyIndexUpdates(IDictionary <string, IMemberUpdate> updates,
bool updateIndexesEagerly,
bool onlyUniqueIndexesWereUpdated,
int numberOfUniqueIndexesUpdated,
bool writeStateIfConstraintsAreNotViolated)
{
//if there is any update to the indexes
//we go ahead and updates the indexes
if (updates.Count() > 0)
{
IList <Type> iGrainTypes = GetIIndexableGrainTypes();
IIndexableGrain thisGrain = this.AsReference <IIndexableGrain>(GrainFactory);
bool isThereAtMostOneUniqueIndex = numberOfUniqueIndexesUpdated <= 1;
//if any unique index is defined on this grain and at least one of them is updated
if (numberOfUniqueIndexesUpdated > 0)
{
try
{
//update the unique indexes eagerly
//if there were more than one unique index, the updates to
//the unique indexes should be tentative in order not to
//become visible to readers before making sure that all
//uniqueness constraints are satisfied
await ApplyIndexUpdatesEagerly(iGrainTypes, thisGrain, updates, true, false, !isThereAtMostOneUniqueIndex);
}
catch (UniquenessConstraintViolatedException ex)
{
//if any uniqueness constraint is violated and we have
//more than one unique index defined, then all tentative
//updates should be undone
if (!isThereAtMostOneUniqueIndex)
{
await UndoTentativeChangesToUniqueIndexesEagerly(iGrainTypes, thisGrain, updates);
}
//then, the exception is thrown back to the user code.
throw ex;
}
}
//if indexes are updated eagerly
if (updateIndexesEagerly)
{
//Case 1: if only unique indexes were updated, then their update
//is already processed before and the only thing remaining is to
//save the grain state if requested
if (onlyUniqueIndexesWereUpdated && writeStateIfConstraintsAreNotViolated)
{
await WriteBaseStateAsync();
}
//Case 2: if there were some non-unique indexes updates and
//writing the state back to the storage is requested, then we
//do these two tasks concurrently
else if (writeStateIfConstraintsAreNotViolated)
{
await Task.WhenAll(
WriteBaseStateAsync(),
ApplyIndexUpdatesEagerly(iGrainTypes, thisGrain, updates, false, isThereAtMostOneUniqueIndex)
);
}
//Case 3: if there were some non-unique indexes updates, but
//writing the state back to the storage is not requested, then
//the only thing left is updating the remaining non-unique indexes
else
{
await ApplyIndexUpdatesEagerly(iGrainTypes, thisGrain, updates, false, isThereAtMostOneUniqueIndex);
}
}
//Otherwise, if indexes are updated lazily
else
{
//update the indexes lazily
ApplyIndexUpdatesLazilyWithoutWait(updates, iGrainTypes, thisGrain, Guid.NewGuid());
//final, the grain state is persisted if requested
if (writeStateIfConstraintsAreNotViolated)
{
await WriteBaseStateAsync();
}
}
//if everything was successful, the before images are updated
UpdateBeforeImages(updates);
}
//otherwise if there is no update to the indexes, we should
//write back the state of the grain if requested
else if (writeStateIfConstraintsAreNotViolated)
{
await WriteBaseStateAsync();
}
}
private Task UndoTentativeChangesToUniqueIndexesEagerly(IList <Type> iGrainTypes,
IIndexableGrain thisGrain,
IDictionary <string, IMemberUpdate> updates)
{
return(ApplyIndexUpdatesEagerly(iGrainTypes, thisGrain, MemberUpdateReverseTentative.Reverse(updates), true, false, false));
}
=> Task.FromResult(true); // The index is maintained by the underlying _grainStorage, when the grain's WriteStateAsync is called
public Task <bool> DirectApplyIndexUpdate(IIndexableGrain g, Immutable <IMemberUpdate> iUpdate, bool isUniqueIndex,
IndexMetaData idxMetaData, SiloAddress siloAddress)
=> Task.FromResult(true); // The index is maintained by the underlying _grainStorage, when the grain's WriteStateAsync is called
