/// <inheritdoc/>
protected override void ProcessNotifications()
{
// discard notifications that are behind our already confirmed state
while (notifications.Count > 0 && notifications.ElementAt(0).Key < GlobalStateCache.GlobalVersion)
{
Services.Log(Severity.Verbose, "discarding notification {0}", notifications.ElementAt(0).Value);
notifications.RemoveAt(0);
}
// process notifications that reflect next global version
while (notifications.Count > 0 && notifications.ElementAt(0).Key == GlobalStateCache.GlobalVersion)
{
var updateNotification = notifications.ElementAt(0).Value;
notifications.RemoveAt(0);
// Apply all operations in pending
foreach (var u in updateNotification.Updates)
{
try
{
Host.UpdateView(GlobalStateCache.State, u);
}
catch (Exception e)
{
Services.CaughtUserCodeException("UpdateView", nameof(ProcessNotifications), e);
}
}
GlobalStateCache.GlobalVersion = updateNotification.Version;
GlobalStateCache.FlipBit(updateNotification.Origin);
GlobalStateCache.ETag = updateNotification.ETag;
Services.Log(Severity.Verbose, "notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalStateCache);
}
Services.Log(Severity.Verbose2, "unprocessed notifications in queue: {0}", notifications.Count);
base.ProcessNotifications();
}
/// <inheritdoc/>
protected override async Task <int> WriteAsync()
{
enter_operation("WriteAsync");
var state = CopyTentativeState();
var updates = GetCurrentBatchOfUpdates();
bool batchsuccessfullywritten = false;
var nextglobalstate = new GrainStateWithMetaDataAndETag <TLogView>(state);
nextglobalstate.WriteVector = GlobalStateCache.WriteVector;
nextglobalstate.GlobalVersion = GlobalStateCache.GlobalVersion + updates.Length;
nextglobalstate.ETag = GlobalStateCache.ETag;
var writebit = nextglobalstate.FlipBit(Services.MyClusterId);
try
{
// for manual testing
//await Task.Delay(5000);
await globalStorageProvider.WriteStateAsync(grainTypeName, Services.GrainReference, nextglobalstate);
batchsuccessfullywritten = true;
GlobalStateCache = nextglobalstate;
Services.Log(Severity.Verbose, "write ({0} updates) success {1}", updates.Length, GlobalStateCache);
LastPrimaryIssue.Resolve(Host, Services);
}
catch (Exception e)
{
LastPrimaryIssue.Record(new UpdateStateStorageFailed()
{
Exception = e
}, Host, Services);
}
if (!batchsuccessfullywritten)
{
Services.Log(Severity.Verbose, "write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue);
while (true) // be stubborn until we can read what is there
{
await LastPrimaryIssue.DelayBeforeRetry();
try
{
await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache);
Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache);
LastPrimaryIssue.Resolve(Host, Services);
break;
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue);
}
// check if last apparently failed write was in fact successful
if (writebit == GlobalStateCache.GetBit(Services.MyClusterId))
{
GlobalStateCache = nextglobalstate;
Services.Log(Severity.Verbose, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache);
batchsuccessfullywritten = true;
}
}
// broadcast notifications to all other clusters
if (batchsuccessfullywritten)
{
BroadcastNotification(new UpdateNotificationMessage()
{
Version = GlobalStateCache.GlobalVersion,
Updates = updates.Select(se => se.Entry).ToList(),
Origin = Services.MyClusterId,
ETag = GlobalStateCache.ETag
});
}
exit_operation("WriteAsync");
if (!batchsuccessfullywritten)
{
return(0);
}
return(updates.Length);
}