/// <inheritdoc/>
protected override async Task ReadAsync()
{
enter_operation("ReadAsync");
while (true)
{
try
{
// for manual testing
//await Task.Delay(5000);
await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache);
Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache);
LastPrimaryIssue.Resolve(Host, Services);
break; // successful
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue);
await LastPrimaryIssue.DelayBeforeRetry();
}
exit_operation("ReadAsync");
}
protected override async Task ReadAsync()
{
enter_operation("ReadAsync");
while (true)
{
try
{
await _grainStorage.ReadStateAsync(_grainTypeName, Services.GrainReference, _snapshotState);
await EnsureStateGlobalVersion();
Services.Log(LogLevel.Debug, "read success {0}", _snapshotState);
if (_confirmedVersionInternal < _snapshotState.StateAndMetaData.SnapshotVersion)
{
_confirmedVersionInternal = _snapshotState.StateAndMetaData.SnapshotVersion;
_confirmedViewInternal = _snapshotState.StateAndMetaData.Snapshot.DeepClone();
}
var logs = await RetrieveLogSegment(
_confirmedVersionInternal,
_snapshotState.StateAndMetaData.GlobalVersion);
Services.Log(LogLevel.Debug, "read success {0}", logs);
UpdateConfirmedView(logs);
LastPrimaryIssue.Resolve(Host, Services);
break; // successful
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromSnapshotStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
await LastPrimaryIssue.DelayBeforeRetry();
}
exit_operation("ReadAsync");
}
/// <inheritdoc/>
protected override async Task ReadAsync()
{
enter_operation("ReadAsync");
while (true)
{
try
{
// read from storage
var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage();
version = result.Key;
cached = result.Value;
Services.Log(LogLevel.Debug, "read success v{0}", version);
LastPrimaryIssue.Resolve(Host, Services);
break; // successful
}
catch (Exception e)
{
// unwrap inner exception that was forwarded - helpful for debugging
if ((e as ProtocolTransportException)?.InnerException != null)
{
e = ((ProtocolTransportException)e).InnerException;
}
LastPrimaryIssue.Record(new ReadFromPrimaryFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
await LastPrimaryIssue.DelayBeforeRetry();
}
exit_operation("ReadAsync");
}
/// <inheritdoc/>
protected override async Task ReadAsync()
{
enter_operation("ReadAsync");
while (true)
{
try
{
// for manual testing
//await Task.Delay(5000);
var readState = new GrainStateWithMetaDataAndETag <TLogView>();
await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, readState);
GlobalStateCache = readState;
Services.Log(LogLevel.Debug, "read success {0}", GlobalStateCache);
LastPrimaryIssue.Resolve(Host, Services);
break; // successful
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
await LastPrimaryIssue.DelayBeforeRetry();
}
exit_operation("ReadAsync");
}
/// <inheritdoc/>
protected override async Task ReadAsync()
{
enter_operation("ReadAsync");
while (true)
{
try
{
// for manual testing
//await Task.Delay(5000);
await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog);
Services.Log(LogLevel.Debug, "read success {0}", GlobalLog);
UpdateConfirmedView();
LastPrimaryIssue.Resolve(Host, Services);
break; // successful
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromLogStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
await LastPrimaryIssue.DelayBeforeRetry();
}
exit_operation("ReadAsync");
}
protected override async Task <int> WriteAsync()
{
enter_operation("WriteAsync");
var updates = GetCurrentBatchOfUpdates();
bool batchSuccessfullyWritten = false;
bool logsSuccessfullySaved = false;
var writebit = _snapshotState.StateAndMetaData.FlipBit(Services.MyClusterId);
if (!_useIndependentEventStorage)
{
foreach (var x in updates)
{
_snapshotState.StateAndMetaData.Log.Add(x.Entry);
}
_snapshotState.StateAndMetaData.GlobalVersion += updates.Length;
logsSuccessfullySaved = true;
}
else
{
try
{
int expectedVersion = _snapshotState.StateAndMetaData.GlobalVersion + updates.Length;
await _eventStorage.SaveEvents(
_grainTypeName,
Services.GrainReference,
updates.Select(x => x.Entry),
expectedVersion - _snapshotState.StateAndMetaData.Log.Count);
_snapshotState.StateAndMetaData.GlobalVersion = expectedVersion;
logsSuccessfullySaved = true;
}
catch (Exception e)
{
LastPrimaryIssue.Record(new UpdateEventStorageFailed()
{
Exception = e
}, Host, Services);
}
}
if (logsSuccessfullySaved)
{
try
{
if (_snapshotStrategy(GetSnapshotStrategyInfo()))
{
_snapshotState.StateAndMetaData.SnapshotVersion = _confirmedVersionInternal;
_snapshotState.StateAndMetaData.SnapshotUpdatedTime = DateTime.Now;
_snapshotState.StateAndMetaData.Snapshot = _confirmedViewInternal.DeepClone();
}
await _grainStorage.WriteStateAsync(_grainTypeName, Services.GrainReference, _snapshotState);
batchSuccessfullyWritten = true;
Services.Log(LogLevel.Debug, "write ({0} updates) success {1}", updates.Length, _snapshotState);
UpdateConfirmedView(updates.Select(u => u.Entry));
LastPrimaryIssue.Resolve(Host, Services);
}
catch (Exception e)
{
LastPrimaryIssue.Record(new UpdateSnapshotStorageFailed()
{
Exception = e
}, Host, Services);
}
}
if (!batchSuccessfullyWritten)
{
Services.Log(LogLevel.Debug, "write apparently failed {0}", LastPrimaryIssue);
while (true)
{
await LastPrimaryIssue.DelayBeforeRetry();
try
{
await _grainStorage.ReadStateAsync(_grainTypeName, Services.GrainReference, _snapshotState);
Services.Log(LogLevel.Debug, "read success {0}", _snapshotState);
if (_confirmedVersionInternal < _snapshotState.StateAndMetaData.SnapshotVersion)
{
_confirmedVersionInternal = _snapshotState.StateAndMetaData.SnapshotVersion;
_confirmedViewInternal = _snapshotState.StateAndMetaData.Snapshot;
}
var logs = await RetrieveLogSegment(
_confirmedVersionInternal,
_snapshotState.StateAndMetaData.GlobalVersion);
Services.Log(LogLevel.Debug, "read success {0}", logs);
UpdateConfirmedView(logs);
LastPrimaryIssue.Resolve(Host, Services);
break;
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromSnapshotStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
}
if (writebit == _snapshotState.StateAndMetaData.GetBit(Services.MyClusterId))
{
Services.Log(LogLevel.Debug, "last write ({0} updates) was actually a success {1}", updates.Length, _snapshotState);
batchSuccessfullyWritten = true;
}
}
exit_operation("WriteAsync");
if (!batchSuccessfullyWritten)
{
return(0);
}
return(updates.Length);
}
/// <inheritdoc/>
protected override async Task <int> WriteAsync()
{
enter_operation("WriteAsync");
var state = CopyTentativeState();
var updates = GetCurrentBatchOfUpdates();
bool batchsuccessfullywritten = false;
var nextglobalstate = new GrainStateWithMetaDataAndETag <TLogView>(state);
nextglobalstate.StateAndMetaData.WriteVector = GlobalStateCache.StateAndMetaData.WriteVector;
nextglobalstate.StateAndMetaData.GlobalVersion = GlobalStateCache.StateAndMetaData.GlobalVersion + updates.Length;
nextglobalstate.ETag = GlobalStateCache.ETag;
var writebit = nextglobalstate.StateAndMetaData.FlipBit(Services.MyClusterId);
try
{
// for manual testing
//await Task.Delay(5000);
await globalStorageProvider.WriteStateAsync(grainTypeName, Services.GrainReference, nextglobalstate);
batchsuccessfullywritten = true;
GlobalStateCache = nextglobalstate;
Services.Log(Severity.Verbose, "write ({0} updates) success {1}", updates.Length, GlobalStateCache);
LastPrimaryIssue.Resolve(Host, Services);
}
catch (Exception e)
{
LastPrimaryIssue.Record(new UpdateStateStorageFailed()
{
Exception = e
}, Host, Services);
}
if (!batchsuccessfullywritten)
{
Services.Log(Severity.Verbose, "write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue);
while (true) // be stubborn until we can read what is there
{
await LastPrimaryIssue.DelayBeforeRetry();
try
{
await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache);
Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache);
LastPrimaryIssue.Resolve(Host, Services);
break;
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue);
}
// check if last apparently failed write was in fact successful
if (writebit == GlobalStateCache.StateAndMetaData.GetBit(Services.MyClusterId))
{
GlobalStateCache = nextglobalstate;
Services.Log(Severity.Verbose, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache);
batchsuccessfullywritten = true;
}
}
// broadcast notifications to all other clusters
if (batchsuccessfullywritten)
{
BroadcastNotification(new UpdateNotificationMessage()
{
Version = GlobalStateCache.StateAndMetaData.GlobalVersion,
Updates = updates.Select(se => se.Entry).ToList(),
Origin = Services.MyClusterId,
ETag = GlobalStateCache.ETag
});
}
exit_operation("WriteAsync");
if (!batchsuccessfullywritten)
{
return(0);
}
return(updates.Length);
}
/// <inheritdoc/>
protected override async Task <int> WriteAsync()
{
enter_operation("WriteAsync");
var updates = GetCurrentBatchOfUpdates().Select(submissionentry => submissionentry.Entry).ToList();
bool writesuccessful = false;
bool transitionssuccessful = false;
try
{
writesuccessful = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ApplyUpdatesToStorage(updates, version);
LastPrimaryIssue.Resolve(Host, Services);
}
catch (Exception e)
{
// unwrap inner exception that was forwarded - helpful for debugging
if ((e as ProtocolTransportException)?.InnerException != null)
{
e = ((ProtocolTransportException)e).InnerException;
}
LastPrimaryIssue.Record(new UpdatePrimaryFailed()
{
Exception = e
}, Host, Services);
}
if (writesuccessful)
{
Services.Log(LogLevel.Debug, "write ({0} updates) success v{1}", updates.Count, version + updates.Count);
// now we update the cached state by applying the same updates
// in case we encounter any exceptions we will re-read the whole state from storage
try
{
foreach (var u in updates)
{
version++;
Host.UpdateView(this.cached, u);
}
transitionssuccessful = true;
}
catch (Exception e)
{
Services.CaughtUserCodeException("UpdateView", nameof(WriteAsync), e);
}
}
if (!writesuccessful || !transitionssuccessful)
{
Services.Log(LogLevel.Debug, "{0} failed {1}", writesuccessful ? "transitions" : "write", LastPrimaryIssue);
while (true) // be stubborn until we can re-read the state from storage
{
await LastPrimaryIssue.DelayBeforeRetry();
try
{
var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage();
version = result.Key;
cached = result.Value;
Services.Log(LogLevel.Debug, "read success v{0}", version);
LastPrimaryIssue.Resolve(Host, Services);
break;
}
catch (Exception e)
{
// unwrap inner exception that was forwarded - helpful for debugging
if ((e as ProtocolTransportException)?.InnerException != null)
{
e = ((ProtocolTransportException)e).InnerException;
}
LastPrimaryIssue.Record(new ReadFromPrimaryFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
}
}
// broadcast notifications to all other clusters
// TODO: send state instead of updates, if smaller
if (writesuccessful)
{
BroadcastNotification(new UpdateNotificationMessage()
{
Version = version,
Updates = updates,
});
}
exit_operation("WriteAsync");
return(writesuccessful ? updates.Count : 0);
}
/// <inheritdoc/>
protected override async Task ReadAsync()
{
enter_operation("ReadAsync");
while (true)
{
try
{
if (MayAccessStorage())
{
// read from storage
var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage();
version = result.Key;
cached = result.Value;
}
else
{
// read from primary cluster
var request = new ReadRequest()
{
KnownVersion = version
};
if (!Services.MultiClusterConfiguration.Clusters.Contains(primaryCluster))
{
throw new ProtocolTransportException("the specified primary cluster is not in the multicluster configuration");
}
var response = (ReadResponse <TLogView>) await Services.SendMessage(request, primaryCluster);
if (response.Version > request.KnownVersion)
{
version = response.Version;
cached = response.Value;
}
}
Services.Log(LogLevel.Debug, "read success v{0}", version);
LastPrimaryIssue.Resolve(Host, Services);
break; // successful
}
catch (Exception e)
{
// unwrap inner exception that was forwarded - helpful for debugging
if ((e as ProtocolTransportException)?.InnerException != null)
{
e = ((ProtocolTransportException)e).InnerException;
}
LastPrimaryIssue.Record(new ReadFromPrimaryFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
await LastPrimaryIssue.DelayBeforeRetry();
}
exit_operation("ReadAsync");
}
/// <inheritdoc/>
protected override async Task <int> WriteAsync()
{
enter_operation("WriteAsync");
var updates = GetCurrentBatchOfUpdates();
bool batchsuccessfullywritten = false;
var writebit = GlobalLog.StateAndMetaData.FlipBit(Services.MyClusterId);
foreach (var x in updates)
{
GlobalLog.StateAndMetaData.Log.Add(x.Entry);
}
try
{
// for manual testing
//await Task.Delay(5000);
await globalGrainStorage.WriteStateAsync(grainTypeName, Services.GrainReference, GlobalLog);
batchsuccessfullywritten = true;
Services.Log(LogLevel.Debug, "write ({0} updates) success {1}", updates.Length, GlobalLog);
UpdateConfirmedView();
LastPrimaryIssue.Resolve(Host, Services);
}
catch (Exception e)
{
LastPrimaryIssue.Record(new UpdateLogStorageFailed()
{
Exception = e
}, Host, Services);
}
if (!batchsuccessfullywritten)
{
Services.Log(LogLevel.Debug, "write apparently failed {0}", LastPrimaryIssue);
while (true) // be stubborn until we can read what is there
{
await LastPrimaryIssue.DelayBeforeRetry();
try
{
await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog);
Services.Log(LogLevel.Debug, "read success {0}", GlobalLog);
UpdateConfirmedView();
LastPrimaryIssue.Resolve(Host, Services);
break;
}
catch (Exception e)
{
LastPrimaryIssue.Record(new ReadFromLogStorageFailed()
{
Exception = e
}, Host, Services);
}
Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
}
// check if last apparently failed write was in fact successful
if (writebit == GlobalLog.StateAndMetaData.GetBit(Services.MyClusterId))
{
Services.Log(LogLevel.Debug, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalLog);
batchsuccessfullywritten = true;
}
}
exit_operation("WriteAsync");
if (!batchsuccessfullywritten)
{
return(0);
}
return(updates.Length);
}
