コード例 #1
0
ファイル: TransactionQueue.cs プロジェクト: ggaller/orleans
        private async Task StorageWork()
        {
            try
            {
                if (problemFlag != TransactionalStatus.Ok)
                {
                    RWLock.AbortExecutingTransactions();

                    // abort all entries in the commit queue
                    foreach (var entry in commitQueue.Elements)
                    {
                        NotifyOfAbort(entry, problemFlag);
                    }
                    commitQueue.Clear();

                    if (problemFlag == TransactionalStatus.StorageConflict)
                    {
                        logger.Debug("deactivating after storage conflict");
                        this.deactivate();
                        this.RWLock.AbortQueuedTransactions();
                    }
                    else
                    {
                        logger.Debug($"restoring state after status={problemFlag}");
                        // recover, clear storageFlag, then allow next queued transaction(s) to enter lock
                        await NotifyOfRestore();
                    }
                }
                else
                {
                    // count committable entries at the bottom of the commit queue
                    int committableEntries = 0;
                    while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
                    {
                        committableEntries++;
                    }

                    // process all committable entries, assembling a storage batch
                    if (committableEntries > 0)
                    {
                        // process all committable entries, adding storage events to the storage batch
                        CollectEventsForBatch(committableEntries);
                        if (problemFlag != TransactionalStatus.Ok)
                        {
                            return;
                        }

                        if (logger.IsEnabled(LogLevel.Debug))
                        {
                            var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
                            logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
                        }
                    }
                    else
                    {
                        // send or re-send messages and detect timeouts
                        CheckProgressOfCommitQueue();
                    }

                    // store the current storage batch, if it is not empty
                    StorageBatch <TState> batchBeingSentToStorage = null;
                    if (this.storageBatch.BatchSize > 0)
                    {
                        // get the next batch in place so it can be filled while we store the old one
                        batchBeingSentToStorage = this.storageBatch;
                        this.storageBatch       = new StorageBatch <TState>(batchBeingSentToStorage);

                        // perform the actual store, and record the e-tag
                        this.storageBatch.ETag = await batchBeingSentToStorage.Store(storage);
                    }

                    if (committableEntries > 0)
                    {
                        // update stable state
                        var lastCommittedEntry = commitQueue[committableEntries - 1];
                        this.stableState          = lastCommittedEntry.State;
                        this.stableSequenceNumber = lastCommittedEntry.SequenceNumber;
                        if (logger.IsEnabled(LogLevel.Trace))
                        {
                            logger.Trace($"Stable state version: {this.stableSequenceNumber}");
                        }


                        // remove committed entries from commit queue
                        commitQueue.RemoveFromFront(committableEntries);
                        storageWorker.Notify();  // we have to re-check for work
                    }

                    if (batchBeingSentToStorage != null)
                    {
                        batchBeingSentToStorage.RunFollowUpActions();
                        storageWorker.Notify();  // we have to re-check for work
                    }
                }
            }
            catch (InconsistentStateException e)
            {
                logger.Warn(888, $"reload from storage triggered by e-tag mismatch {e}");

                problemFlag = TransactionalStatus.StorageConflict;
            }
            catch (Exception e)
            {
                logger.Warn(888, $"exception in storageWorker", e);

                problemFlag = TransactionalStatus.UnknownException;
            } finally
            {
                if (problemFlag == TransactionalStatus.Ok)
                {
                    this.failCounter = 0;
                }
                else
                {
                    // after exceptions, we try again, but with limits
                    if (++failCounter < 10)
                    {
                        await Task.Delay(100);

                        // this restarts the worker, which sees the problem flag and recovers.
                        storageWorker.Notify();
                    }
                    else
                    {
                        // bail out
                        logger.Warn(999, $"storageWorker is bailing out");
                    }
                }
            }
        }
コード例 #2
0
        private async Task StorageWork()
        {
            try
            {
                // count committable entries at the bottom of the commit queue
                int committableEntries = 0;
                while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
                {
                    committableEntries++;
                }

                // process all committable entries, assembling a storage batch
                if (committableEntries > 0)
                {
                    // process all committable entries, adding storage events to the storage batch
                    CollectEventsForBatch(committableEntries);

                    if (logger.IsEnabled(LogLevel.Debug))
                    {
                        var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
                        logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
                    }
                }
                else
                {
                    // send or re-send messages and detect timeouts
                    await CheckProgressOfCommitQueue();
                }

                // store the current storage batch, if it is not empty
                StorageBatch <TState> batchBeingSentToStorage = null;
                if (this.storageBatch.BatchSize > 0)
                {
                    // get the next batch in place so it can be filled while we store the old one
                    batchBeingSentToStorage = this.storageBatch;
                    this.storageBatch       = new StorageBatch <TState>(batchBeingSentToStorage);

                    try
                    {
                        if (await batchBeingSentToStorage.CheckStorePreConditions())
                        {
                            // perform the actual store, and record the e-tag
                            this.storageBatch.ETag = await batchBeingSentToStorage.Store(storage);

                            failCounter = 0;
                        }
                        else
                        {
                            logger.LogWarning("Store pre conditions not met.");
                            await AbortAndRestore(TransactionalStatus.CommitFailure);

                            return;
                        }
                    }
                    catch (InconsistentStateException e)
                    {
                        logger.LogWarning(888, e, "Reload from storage triggered by e-tag mismatch.");
                        await AbortAndRestore(TransactionalStatus.StorageConflict, true);

                        return;
                    }
                    catch (Exception e)
                    {
                        logger.Warn(888, $"Storage exception in storageWorker.", e);
                        await AbortAndRestore(TransactionalStatus.UnknownException);

                        return;
                    }
                }

                if (committableEntries > 0)
                {
                    // update stable state
                    var lastCommittedEntry = commitQueue[committableEntries - 1];
                    this.stableState          = lastCommittedEntry.State;
                    this.stableSequenceNumber = lastCommittedEntry.SequenceNumber;
                    if (logger.IsEnabled(LogLevel.Trace))
                    {
                        logger.Trace($"Stable state version: {this.stableSequenceNumber}");
                    }

                    // remove committed entries from commit queue
                    commitQueue.RemoveFromFront(committableEntries);
                    storageWorker.Notify();  // we have to re-check for work
                }

                if (batchBeingSentToStorage != null)
                {
                    batchBeingSentToStorage.RunFollowUpActions();
                    storageWorker.Notify();  // we have to re-check for work
                }
            }
            catch (Exception e)
            {
                logger.LogWarning(888, e, "Exception in storageWorker.  Retry {FailCounter}", failCounter);
                await AbortAndRestore(TransactionalStatus.UnknownException);
            }
        }