Beispiel #1
0
        /// <inheritdoc/>
        protected override async Task ReadAsync()
        {
            enter_operation("ReadAsync");

            while (true)
            {
                try
                {
                    // for manual testing
                    //await Task.Delay(5000);

                    await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache);

                    Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache);

                    LastPrimaryIssue.Resolve(Host, Services);

                    break; // successful
                }
                catch (Exception e)
                {
                    LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }

                Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue);

                await LastPrimaryIssue.DelayBeforeRetry();
            }

            exit_operation("ReadAsync");
        }
        protected override async Task ReadAsync()
        {
            enter_operation("ReadAsync");

            while (true)
            {
                try
                {
                    await _grainStorage.ReadStateAsync(_grainTypeName, Services.GrainReference, _snapshotState);
                    await EnsureStateGlobalVersion();

                    Services.Log(LogLevel.Debug, "read success {0}", _snapshotState);

                    if (_confirmedVersionInternal < _snapshotState.StateAndMetaData.SnapshotVersion)
                    {
                        _confirmedVersionInternal = _snapshotState.StateAndMetaData.SnapshotVersion;
                        _confirmedViewInternal    = _snapshotState.StateAndMetaData.Snapshot.DeepClone();
                    }

                    var logs = await RetrieveLogSegment(
                        _confirmedVersionInternal,
                        _snapshotState.StateAndMetaData.GlobalVersion);

                    Services.Log(LogLevel.Debug, "read success {0}", logs);

                    UpdateConfirmedView(logs);

                    LastPrimaryIssue.Resolve(Host, Services);

                    break; // successful
                }
                catch (Exception e)
                {
                    LastPrimaryIssue.Record(new ReadFromSnapshotStorageFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }

                Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);

                await LastPrimaryIssue.DelayBeforeRetry();
            }

            exit_operation("ReadAsync");
        }
Beispiel #3
0
        /// <inheritdoc/>
        protected override async Task ReadAsync()
        {
            enter_operation("ReadAsync");

            while (true)
            {
                try
                {
                    // read from storage
                    var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage();
                    version = result.Key;
                    cached  = result.Value;

                    Services.Log(LogLevel.Debug, "read success v{0}", version);

                    LastPrimaryIssue.Resolve(Host, Services);

                    break; // successful
                }
                catch (Exception e)
                {
                    // unwrap inner exception that was forwarded - helpful for debugging
                    if ((e as ProtocolTransportException)?.InnerException != null)
                    {
                        e = ((ProtocolTransportException)e).InnerException;
                    }

                    LastPrimaryIssue.Record(new ReadFromPrimaryFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }

                Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);

                await LastPrimaryIssue.DelayBeforeRetry();
            }

            exit_operation("ReadAsync");
        }
        /// <inheritdoc/>
        protected override async Task ReadAsync()
        {
            enter_operation("ReadAsync");

            while (true)
            {
                try
                {
                    // for manual testing
                    //await Task.Delay(5000);

                    var readState = new GrainStateWithMetaDataAndETag <TLogView>();

                    await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, readState);

                    GlobalStateCache = readState;

                    Services.Log(LogLevel.Debug, "read success {0}", GlobalStateCache);

                    LastPrimaryIssue.Resolve(Host, Services);

                    break; // successful
                }
                catch (Exception e)
                {
                    LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }

                Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);

                await LastPrimaryIssue.DelayBeforeRetry();
            }

            exit_operation("ReadAsync");
        }
Beispiel #5
0
        /// <inheritdoc/>
        protected override async Task ReadAsync()
        {
            enter_operation("ReadAsync");

            while (true)
            {
                try
                {
                    // for manual testing
                    //await Task.Delay(5000);

                    await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog);

                    Services.Log(LogLevel.Debug, "read success {0}", GlobalLog);

                    UpdateConfirmedView();

                    LastPrimaryIssue.Resolve(Host, Services);

                    break; // successful
                }
                catch (Exception e)
                {
                    LastPrimaryIssue.Record(new ReadFromLogStorageFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }

                Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);

                await LastPrimaryIssue.DelayBeforeRetry();
            }

            exit_operation("ReadAsync");
        }
        protected override async Task <int> WriteAsync()
        {
            enter_operation("WriteAsync");

            var  updates = GetCurrentBatchOfUpdates();
            bool batchSuccessfullyWritten = false;
            bool logsSuccessfullySaved    = false;

            var writebit = _snapshotState.StateAndMetaData.FlipBit(Services.MyClusterId);

            if (!_useIndependentEventStorage)
            {
                foreach (var x in updates)
                {
                    _snapshotState.StateAndMetaData.Log.Add(x.Entry);
                }

                _snapshotState.StateAndMetaData.GlobalVersion += updates.Length;
                logsSuccessfullySaved = true;
            }
            else
            {
                try
                {
                    int expectedVersion = _snapshotState.StateAndMetaData.GlobalVersion + updates.Length;

                    await _eventStorage.SaveEvents(
                        _grainTypeName,
                        Services.GrainReference,
                        updates.Select(x => x.Entry),
                        expectedVersion - _snapshotState.StateAndMetaData.Log.Count);

                    _snapshotState.StateAndMetaData.GlobalVersion = expectedVersion;
                    logsSuccessfullySaved = true;
                }
                catch (Exception e)
                {
                    LastPrimaryIssue.Record(new UpdateEventStorageFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }
            }

            if (logsSuccessfullySaved)
            {
                try
                {
                    if (_snapshotStrategy(GetSnapshotStrategyInfo()))
                    {
                        _snapshotState.StateAndMetaData.SnapshotVersion     = _confirmedVersionInternal;
                        _snapshotState.StateAndMetaData.SnapshotUpdatedTime = DateTime.Now;
                        _snapshotState.StateAndMetaData.Snapshot            = _confirmedViewInternal.DeepClone();
                    }

                    await _grainStorage.WriteStateAsync(_grainTypeName, Services.GrainReference, _snapshotState);

                    batchSuccessfullyWritten = true;

                    Services.Log(LogLevel.Debug, "write ({0} updates) success {1}", updates.Length, _snapshotState);

                    UpdateConfirmedView(updates.Select(u => u.Entry));

                    LastPrimaryIssue.Resolve(Host, Services);
                }
                catch (Exception e)
                {
                    LastPrimaryIssue.Record(new UpdateSnapshotStorageFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }
            }

            if (!batchSuccessfullyWritten)
            {
                Services.Log(LogLevel.Debug, "write apparently failed {0}", LastPrimaryIssue);

                while (true)
                {
                    await LastPrimaryIssue.DelayBeforeRetry();

                    try
                    {
                        await _grainStorage.ReadStateAsync(_grainTypeName, Services.GrainReference, _snapshotState);

                        Services.Log(LogLevel.Debug, "read success {0}", _snapshotState);

                        if (_confirmedVersionInternal < _snapshotState.StateAndMetaData.SnapshotVersion)
                        {
                            _confirmedVersionInternal = _snapshotState.StateAndMetaData.SnapshotVersion;
                            _confirmedViewInternal    = _snapshotState.StateAndMetaData.Snapshot;
                        }

                        var logs = await RetrieveLogSegment(
                            _confirmedVersionInternal,
                            _snapshotState.StateAndMetaData.GlobalVersion);

                        Services.Log(LogLevel.Debug, "read success {0}", logs);

                        UpdateConfirmedView(logs);

                        LastPrimaryIssue.Resolve(Host, Services);

                        break;
                    }
                    catch (Exception e)
                    {
                        LastPrimaryIssue.Record(new ReadFromSnapshotStorageFailed()
                        {
                            Exception = e
                        }, Host, Services);
                    }

                    Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
                }

                if (writebit == _snapshotState.StateAndMetaData.GetBit(Services.MyClusterId))
                {
                    Services.Log(LogLevel.Debug, "last write ({0} updates) was actually a success {1}", updates.Length, _snapshotState);

                    batchSuccessfullyWritten = true;
                }
            }

            exit_operation("WriteAsync");

            if (!batchSuccessfullyWritten)
            {
                return(0);
            }

            return(updates.Length);
        }
Beispiel #7
0
        /// <inheritdoc/>
        protected override async Task <int> WriteAsync()
        {
            enter_operation("WriteAsync");

            var  state   = CopyTentativeState();
            var  updates = GetCurrentBatchOfUpdates();
            bool batchsuccessfullywritten = false;

            var nextglobalstate = new GrainStateWithMetaDataAndETag <TLogView>(state);

            nextglobalstate.StateAndMetaData.WriteVector   = GlobalStateCache.StateAndMetaData.WriteVector;
            nextglobalstate.StateAndMetaData.GlobalVersion = GlobalStateCache.StateAndMetaData.GlobalVersion + updates.Length;
            nextglobalstate.ETag = GlobalStateCache.ETag;

            var writebit = nextglobalstate.StateAndMetaData.FlipBit(Services.MyClusterId);

            try
            {
                // for manual testing
                //await Task.Delay(5000);

                await globalStorageProvider.WriteStateAsync(grainTypeName, Services.GrainReference, nextglobalstate);

                batchsuccessfullywritten = true;

                GlobalStateCache = nextglobalstate;

                Services.Log(Severity.Verbose, "write ({0} updates) success {1}", updates.Length, GlobalStateCache);

                LastPrimaryIssue.Resolve(Host, Services);
            }
            catch (Exception e)
            {
                LastPrimaryIssue.Record(new UpdateStateStorageFailed()
                {
                    Exception = e
                }, Host, Services);
            }

            if (!batchsuccessfullywritten)
            {
                Services.Log(Severity.Verbose, "write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue);

                while (true) // be stubborn until we can read what is there
                {
                    await LastPrimaryIssue.DelayBeforeRetry();

                    try
                    {
                        await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache);

                        Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache);

                        LastPrimaryIssue.Resolve(Host, Services);

                        break;
                    }
                    catch (Exception e)
                    {
                        LastPrimaryIssue.Record(new ReadFromStateStorageFailed()
                        {
                            Exception = e
                        }, Host, Services);
                    }

                    Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue);
                }

                // check if last apparently failed write was in fact successful

                if (writebit == GlobalStateCache.StateAndMetaData.GetBit(Services.MyClusterId))
                {
                    GlobalStateCache = nextglobalstate;

                    Services.Log(Severity.Verbose, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache);

                    batchsuccessfullywritten = true;
                }
            }


            // broadcast notifications to all other clusters
            if (batchsuccessfullywritten)
            {
                BroadcastNotification(new UpdateNotificationMessage()
                {
                    Version = GlobalStateCache.StateAndMetaData.GlobalVersion,
                    Updates = updates.Select(se => se.Entry).ToList(),
                    Origin  = Services.MyClusterId,
                    ETag    = GlobalStateCache.ETag
                });
            }

            exit_operation("WriteAsync");

            if (!batchsuccessfullywritten)
            {
                return(0);
            }

            return(updates.Length);
        }
        /// <inheritdoc/>
        protected override async Task <int> WriteAsync()
        {
            enter_operation("WriteAsync");

            var  updates               = GetCurrentBatchOfUpdates().Select(submissionentry => submissionentry.Entry).ToList();
            bool writesuccessful       = false;
            bool transitionssuccessful = false;

            try
            {
                writesuccessful = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ApplyUpdatesToStorage(updates, version);

                LastPrimaryIssue.Resolve(Host, Services);
            }
            catch (Exception e)
            {
                // unwrap inner exception that was forwarded - helpful for debugging
                if ((e as ProtocolTransportException)?.InnerException != null)
                {
                    e = ((ProtocolTransportException)e).InnerException;
                }

                LastPrimaryIssue.Record(new UpdatePrimaryFailed()
                {
                    Exception = e
                }, Host, Services);
            }

            if (writesuccessful)
            {
                Services.Log(LogLevel.Debug, "write ({0} updates) success v{1}", updates.Count, version + updates.Count);

                // now we update the cached state by applying the same updates
                // in case we encounter any exceptions we will re-read the whole state from storage
                try
                {
                    foreach (var u in updates)
                    {
                        version++;
                        Host.UpdateView(this.cached, u);
                    }

                    transitionssuccessful = true;
                }
                catch (Exception e)
                {
                    Services.CaughtUserCodeException("UpdateView", nameof(WriteAsync), e);
                }
            }

            if (!writesuccessful || !transitionssuccessful)
            {
                Services.Log(LogLevel.Debug, "{0} failed {1}", writesuccessful ? "transitions" : "write", LastPrimaryIssue);

                while (true) // be stubborn until we can re-read the state from storage
                {
                    await LastPrimaryIssue.DelayBeforeRetry();

                    try
                    {
                        var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage();
                        version = result.Key;
                        cached  = result.Value;

                        Services.Log(LogLevel.Debug, "read success v{0}", version);

                        LastPrimaryIssue.Resolve(Host, Services);

                        break;
                    }
                    catch (Exception e)
                    {
                        // unwrap inner exception that was forwarded - helpful for debugging
                        if ((e as ProtocolTransportException)?.InnerException != null)
                        {
                            e = ((ProtocolTransportException)e).InnerException;
                        }

                        LastPrimaryIssue.Record(new ReadFromPrimaryFailed()
                        {
                            Exception = e
                        }, Host, Services);
                    }

                    Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
                }
            }

            // broadcast notifications to all other clusters
            // TODO: send state instead of updates, if smaller
            if (writesuccessful)
            {
                BroadcastNotification(new UpdateNotificationMessage()
                {
                    Version = version,
                    Updates = updates,
                });
            }

            exit_operation("WriteAsync");

            return(writesuccessful ? updates.Count : 0);
        }
        /// <inheritdoc/>
        protected override async Task ReadAsync()
        {
            enter_operation("ReadAsync");

            while (true)
            {
                try
                {
                    if (MayAccessStorage())
                    {
                        // read from storage
                        var result = await((ICustomStorageInterface <TLogView, TLogEntry>)Host).ReadStateFromStorage();
                        version = result.Key;
                        cached  = result.Value;
                    }
                    else
                    {
                        // read from primary cluster
                        var request = new ReadRequest()
                        {
                            KnownVersion = version
                        };
                        if (!Services.MultiClusterConfiguration.Clusters.Contains(primaryCluster))
                        {
                            throw new ProtocolTransportException("the specified primary cluster is not in the multicluster configuration");
                        }
                        var response = (ReadResponse <TLogView>) await Services.SendMessage(request, primaryCluster);

                        if (response.Version > request.KnownVersion)
                        {
                            version = response.Version;
                            cached  = response.Value;
                        }
                    }

                    Services.Log(LogLevel.Debug, "read success v{0}", version);

                    LastPrimaryIssue.Resolve(Host, Services);

                    break; // successful
                }
                catch (Exception e)
                {
                    // unwrap inner exception that was forwarded - helpful for debugging
                    if ((e as ProtocolTransportException)?.InnerException != null)
                    {
                        e = ((ProtocolTransportException)e).InnerException;
                    }

                    LastPrimaryIssue.Record(new ReadFromPrimaryFailed()
                    {
                        Exception = e
                    }, Host, Services);
                }

                Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);

                await LastPrimaryIssue.DelayBeforeRetry();
            }

            exit_operation("ReadAsync");
        }
Beispiel #10
0
        /// <inheritdoc/>
        protected override async Task <int> WriteAsync()
        {
            enter_operation("WriteAsync");

            var  updates = GetCurrentBatchOfUpdates();
            bool batchsuccessfullywritten = false;

            var writebit = GlobalLog.StateAndMetaData.FlipBit(Services.MyClusterId);

            foreach (var x in updates)
            {
                GlobalLog.StateAndMetaData.Log.Add(x.Entry);
            }

            try
            {
                // for manual testing
                //await Task.Delay(5000);

                await globalGrainStorage.WriteStateAsync(grainTypeName, Services.GrainReference, GlobalLog);

                batchsuccessfullywritten = true;

                Services.Log(LogLevel.Debug, "write ({0} updates) success {1}", updates.Length, GlobalLog);

                UpdateConfirmedView();

                LastPrimaryIssue.Resolve(Host, Services);
            }
            catch (Exception e)
            {
                LastPrimaryIssue.Record(new UpdateLogStorageFailed()
                {
                    Exception = e
                }, Host, Services);
            }

            if (!batchsuccessfullywritten)
            {
                Services.Log(LogLevel.Debug, "write apparently failed {0}", LastPrimaryIssue);

                while (true) // be stubborn until we can read what is there
                {
                    await LastPrimaryIssue.DelayBeforeRetry();

                    try
                    {
                        await globalGrainStorage.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog);

                        Services.Log(LogLevel.Debug, "read success {0}", GlobalLog);

                        UpdateConfirmedView();

                        LastPrimaryIssue.Resolve(Host, Services);

                        break;
                    }
                    catch (Exception e)
                    {
                        LastPrimaryIssue.Record(new ReadFromLogStorageFailed()
                        {
                            Exception = e
                        }, Host, Services);
                    }

                    Services.Log(LogLevel.Debug, "read failed {0}", LastPrimaryIssue);
                }

                // check if last apparently failed write was in fact successful

                if (writebit == GlobalLog.StateAndMetaData.GetBit(Services.MyClusterId))
                {
                    Services.Log(LogLevel.Debug, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalLog);

                    batchsuccessfullywritten = true;
                }
            }

            exit_operation("WriteAsync");

            if (!batchsuccessfullywritten)
            {
                return(0);
            }

            return(updates.Length);
        }