/// <inheritdoc/> protected override async Task ReadAsync() { enter_operation("ReadAsync"); while (true) { try { // for manual testing //await Task.Delay(5000); await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache); Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); break; // successful } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } exit_operation("ReadAsync"); }
protected override async Task ReadAsync() { enter_operation("ReadAsync"); while (true) { try { await _grainStorage.ReadStateAsync(_grainTypeName, Services.GrainReference, _snapshotState); await EnsureStateGlobalVersion(); Services.Log(LogLevel.Debug, "read success {0}", _snapshotState); if (_confirmedVersionInternal < _snapshotState.StateAndMetaData.SnapshotVersion) { _confirmedVersionInternal = _snapshotState.StateAndMetaData.SnapshotVersion; _confirmedViewInternal = _snapshotState.StateAndMetaData.Snapshot.DeepClone(); } var logs = await RetrieveLogSegment( _confirmedVersionInternal, _snapshotState.StateAndMetaData.GlobalVersion); Services.Log(LogLevel.Debug, "read success {0}", logs); UpdateConfirmedView(logs); LastPrimaryIssue.Resolve(Host, Services); break; // successful } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromSnapshotStorageFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } exit_operation("ReadAsync"); }
/// <inheritdoc/> protected override async Task ReadAsync() { enter_operation("ReadAsync"); while (true) { try { // read from storage var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage(); version = result.Key; cached = result.Value; Services.Log(LogLevel.Debug, "read success v{0}", version); LastPrimaryIssue.Resolve(Host, Services); break; // successful } catch (Exception e) { // unwrap inner exception that was forwarded - helpful for debugging if ((e as ProtocolTransportException)?.InnerException != null) { e = ((ProtocolTransportException)e).InnerException; } LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } exit_operation("ReadAsync"); }
/// <inheritdoc/> protected override async Task ReadAsync() { enter_operation("ReadAsync"); while (true) { try { // for manual testing //await Task.Delay(5000); var readState = new GrainStateWithMetaDataAndETag <TLogView>(); await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, readState); GlobalStateCache = readState; Services.Log(LogLevel.Debug, "read success {0}", GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); break; // successful } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } exit_operation("ReadAsync"); }
/// <inheritdoc/> protected override async Task ReadAsync() { enter_operation("ReadAsync"); while (true) { try { // for manual testing //await Task.Delay(5000); await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog); Services.Log(LogLevel.Debug, "read success {0}", GlobalLog); UpdateConfirmedView(); LastPrimaryIssue.Resolve(Host, Services); break; // successful } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromLogStorageFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } exit_operation("ReadAsync"); }
protected override async Task <int> WriteAsync() { enter_operation("WriteAsync"); var updates = GetCurrentBatchOfUpdates(); bool batchSuccessfullyWritten = false; bool logsSuccessfullySaved = false; var writebit = _snapshotState.StateAndMetaData.FlipBit(Services.MyClusterId); if (!_useIndependentEventStorage) { foreach (var x in updates) { _snapshotState.StateAndMetaData.Log.Add(x.Entry); } _snapshotState.StateAndMetaData.GlobalVersion += updates.Length; logsSuccessfullySaved = true; } else { try { int expectedVersion = _snapshotState.StateAndMetaData.GlobalVersion + updates.Length; await _eventStorage.SaveEvents( _grainTypeName, Services.GrainReference, updates.Select(x => x.Entry), expectedVersion - _snapshotState.StateAndMetaData.Log.Count); _snapshotState.StateAndMetaData.GlobalVersion = expectedVersion; logsSuccessfullySaved = true; } catch (Exception e) { LastPrimaryIssue.Record(new UpdateEventStorageFailed() { Exception = e }, Host, Services); } } if (logsSuccessfullySaved) { try { if (_snapshotStrategy(GetSnapshotStrategyInfo())) { _snapshotState.StateAndMetaData.SnapshotVersion = _confirmedVersionInternal; _snapshotState.StateAndMetaData.SnapshotUpdatedTime = DateTime.Now; _snapshotState.StateAndMetaData.Snapshot = _confirmedViewInternal.DeepClone(); } await _grainStorage.WriteStateAsync(_grainTypeName, Services.GrainReference, _snapshotState); batchSuccessfullyWritten = true; Services.Log(LogLevel.Debug, "write ({0} updates) success {1}", updates.Length, _snapshotState); UpdateConfirmedView(updates.Select(u => u.Entry)); LastPrimaryIssue.Resolve(Host, Services); } catch (Exception e) { LastPrimaryIssue.Record(new UpdateSnapshotStorageFailed() { Exception = e }, Host, Services); } } if (!batchSuccessfullyWritten) { Services.Log(LogLevel.Debug, "write apparently failed {0}", LastPrimaryIssue); while (true) { await LastPrimaryIssue.DelayBeforeRetry(); try { await _grainStorage.ReadStateAsync(_grainTypeName, Services.GrainReference, _snapshotState); Services.Log(LogLevel.Debug, "read success {0}", _snapshotState); if (_confirmedVersionInternal < _snapshotState.StateAndMetaData.SnapshotVersion) { _confirmedVersionInternal = _snapshotState.StateAndMetaData.SnapshotVersion; _confirmedViewInternal = _snapshotState.StateAndMetaData.Snapshot; } var logs = await RetrieveLogSegment( _confirmedVersionInternal, _snapshotState.StateAndMetaData.GlobalVersion); Services.Log(LogLevel.Debug, "read success {0}", logs); UpdateConfirmedView(logs); LastPrimaryIssue.Resolve(Host, Services); break; } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromSnapshotStorageFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); } if (writebit == _snapshotState.StateAndMetaData.GetBit(Services.MyClusterId)) { Services.Log(LogLevel.Debug, "last write ({0} updates) was actually a success {1}", updates.Length, _snapshotState); batchSuccessfullyWritten = true; } } exit_operation("WriteAsync"); if (!batchSuccessfullyWritten) { return(0); } return(updates.Length); }
/// <inheritdoc/> protected override async Task <int> WriteAsync() { enter_operation("WriteAsync"); var state = CopyTentativeState(); var updates = GetCurrentBatchOfUpdates(); bool batchsuccessfullywritten = false; var nextglobalstate = new GrainStateWithMetaDataAndETag <TLogView>(state); nextglobalstate.StateAndMetaData.WriteVector = GlobalStateCache.StateAndMetaData.WriteVector; nextglobalstate.StateAndMetaData.GlobalVersion = GlobalStateCache.StateAndMetaData.GlobalVersion + updates.Length; nextglobalstate.ETag = GlobalStateCache.ETag; var writebit = nextglobalstate.StateAndMetaData.FlipBit(Services.MyClusterId); try { // for manual testing //await Task.Delay(5000); await globalStorageProvider.WriteStateAsync(grainTypeName, Services.GrainReference, nextglobalstate); batchsuccessfullywritten = true; GlobalStateCache = nextglobalstate; Services.Log(Severity.Verbose, "write ({0} updates) success {1}", updates.Length, GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); } catch (Exception e) { LastPrimaryIssue.Record(new UpdateStateStorageFailed() { Exception = e }, Host, Services); } if (!batchsuccessfullywritten) { Services.Log(Severity.Verbose, "write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue); while (true) // be stubborn until we can read what is there { await LastPrimaryIssue.DelayBeforeRetry(); try { await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache); Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); break; } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); } // check if last apparently failed write was in fact successful if (writebit == GlobalStateCache.StateAndMetaData.GetBit(Services.MyClusterId)) { GlobalStateCache = nextglobalstate; Services.Log(Severity.Verbose, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache); batchsuccessfullywritten = true; } } // broadcast notifications to all other clusters if (batchsuccessfullywritten) { BroadcastNotification(new UpdateNotificationMessage() { Version = GlobalStateCache.StateAndMetaData.GlobalVersion, Updates = updates.Select(se => se.Entry).ToList(), Origin = Services.MyClusterId, ETag = GlobalStateCache.ETag }); } exit_operation("WriteAsync"); if (!batchsuccessfullywritten) { return(0); } return(updates.Length); }
/// <inheritdoc/> protected override async Task <int> WriteAsync() { enter_operation("WriteAsync"); var updates = GetCurrentBatchOfUpdates().Select(submissionentry => submissionentry.Entry).ToList(); bool writesuccessful = false; bool transitionssuccessful = false; try { writesuccessful = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ApplyUpdatesToStorage(updates, version); LastPrimaryIssue.Resolve(Host, Services); } catch (Exception e) { // unwrap inner exception that was forwarded - helpful for debugging if ((e as ProtocolTransportException)?.InnerException != null) { e = ((ProtocolTransportException)e).InnerException; } LastPrimaryIssue.Record(new UpdatePrimaryFailed() { Exception = e }, Host, Services); } if (writesuccessful) { Services.Log(LogLevel.Debug, "write ({0} updates) success v{1}", updates.Count, version + updates.Count); // now we update the cached state by applying the same updates // in case we encounter any exceptions we will re-read the whole state from storage try { foreach (var u in updates) { version++; Host.UpdateView(this.cached, u); } transitionssuccessful = true; } catch (Exception e) { Services.CaughtUserCodeException("UpdateView", nameof(WriteAsync), e); } } if (!writesuccessful || !transitionssuccessful) { Services.Log(LogLevel.Debug, "{0} failed {1}", writesuccessful ? "transitions" : "write", LastPrimaryIssue); while (true) // be stubborn until we can re-read the state from storage { await LastPrimaryIssue.DelayBeforeRetry(); try { var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage(); version = result.Key; cached = result.Value; Services.Log(LogLevel.Debug, "read success v{0}", version); LastPrimaryIssue.Resolve(Host, Services); break; } catch (Exception e) { // unwrap inner exception that was forwarded - helpful for debugging if ((e as ProtocolTransportException)?.InnerException != null) { e = ((ProtocolTransportException)e).InnerException; } LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); } } // broadcast notifications to all other clusters // TODO: send state instead of updates, if smaller if (writesuccessful) { BroadcastNotification(new UpdateNotificationMessage() { Version = version, Updates = updates, }); } exit_operation("WriteAsync"); return(writesuccessful ? updates.Count : 0); }
/// <inheritdoc/> protected override async Task ReadAsync() { enter_operation("ReadAsync"); while (true) { try { if (MayAccessStorage()) { // read from storage var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage(); version = result.Key; cached = result.Value; } else { // read from primary cluster var request = new ReadRequest() { KnownVersion = version }; if (!Services.MultiClusterConfiguration.Clusters.Contains(primaryCluster)) { throw new ProtocolTransportException("the specified primary cluster is not in the multicluster configuration"); } var response = (ReadResponse <TLogView>) await Services.SendMessage(request, primaryCluster); if (response.Version > request.KnownVersion) { version = response.Version; cached = response.Value; } } Services.Log(LogLevel.Debug, "read success v{0}", version); LastPrimaryIssue.Resolve(Host, Services); break; // successful } catch (Exception e) { // unwrap inner exception that was forwarded - helpful for debugging if ((e as ProtocolTransportException)?.InnerException != null) { e = ((ProtocolTransportException)e).InnerException; } LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } exit_operation("ReadAsync"); }
/// <inheritdoc/> protected override async Task <int> WriteAsync() { enter_operation("WriteAsync"); var updates = GetCurrentBatchOfUpdates(); bool batchsuccessfullywritten = false; var writebit = GlobalLog.StateAndMetaData.FlipBit(Services.MyClusterId); foreach (var x in updates) { GlobalLog.StateAndMetaData.Log.Add(x.Entry); } try { // for manual testing //await Task.Delay(5000); await globalGrainStorage.WriteStateAsync(grainTypeName, Services.GrainReference, GlobalLog); batchsuccessfullywritten = true; Services.Log(LogLevel.Debug, "write ({0} updates) success {1}", updates.Length, GlobalLog); UpdateConfirmedView(); LastPrimaryIssue.Resolve(Host, Services); } catch (Exception e) { LastPrimaryIssue.Record(new UpdateLogStorageFailed() { Exception = e }, Host, Services); } if (!batchsuccessfullywritten) { Services.Log(LogLevel.Debug, "write apparently failed {0}", LastPrimaryIssue); while (true) // be stubborn until we can read what is there { await LastPrimaryIssue.DelayBeforeRetry(); try { await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog); Services.Log(LogLevel.Debug, "read success {0}", GlobalLog); UpdateConfirmedView(); LastPrimaryIssue.Resolve(Host, Services); break; } catch (Exception e) { LastPrimaryIssue.Record(new ReadFromLogStorageFailed() { Exception = e }, Host, Services); } Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue); } // check if last apparently failed write was in fact successful if (writebit == GlobalLog.StateAndMetaData.GetBit(Services.MyClusterId)) { Services.Log(LogLevel.Debug, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalLog); batchsuccessfullywritten = true; } } exit_operation("WriteAsync"); if (!batchsuccessfullywritten) { return(0); } return(updates.Length); }