private async Task StorageWork()
{
try
{
if (problemFlag != TransactionalStatus.Ok)
{
RWLock.AbortExecutingTransactions();
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
NotifyOfAbort(entry, problemFlag);
}
commitQueue.Clear();
if (problemFlag == TransactionalStatus.StorageConflict)
{
logger.Debug("deactivating after storage conflict");
this.deactivate();
this.RWLock.AbortQueuedTransactions();
}
else
{
logger.Debug($"restoring state after status={problemFlag}");
// recover, clear storageFlag, then allow next queued transaction(s) to enter lock
await NotifyOfRestore();
}
}
else
{
// count committable entries at the bottom of the commit queue
int committableEntries = 0;
while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
{
committableEntries++;
}
// process all committable entries, assembling a storage batch
if (committableEntries > 0)
{
// process all committable entries, adding storage events to the storage batch
CollectEventsForBatch(committableEntries);
if (problemFlag != TransactionalStatus.Ok)
{
return;
}
if (logger.IsEnabled(LogLevel.Debug))
{
var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
}
}
else
{
// send or re-send messages and detect timeouts
CheckProgressOfCommitQueue();
}
// store the current storage batch, if it is not empty
StorageBatch <TState> batchBeingSentToStorage = null;
if (this.storageBatch.BatchSize > 0)
{
// get the next batch in place so it can be filled while we store the old one
batchBeingSentToStorage = this.storageBatch;
this.storageBatch = new StorageBatch <TState>(batchBeingSentToStorage);
// perform the actual store, and record the e-tag
this.storageBatch.ETag = await batchBeingSentToStorage.Store(storage);
}
if (committableEntries > 0)
{
// update stable state
var lastCommittedEntry = commitQueue[committableEntries - 1];
this.stableState = lastCommittedEntry.State;
this.stableSequenceNumber = lastCommittedEntry.SequenceNumber;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"Stable state version: {this.stableSequenceNumber}");
}
// remove committed entries from commit queue
commitQueue.RemoveFromFront(committableEntries);
storageWorker.Notify(); // we have to re-check for work
}
if (batchBeingSentToStorage != null)
{
batchBeingSentToStorage.RunFollowUpActions();
storageWorker.Notify(); // we have to re-check for work
}
}
}
catch (InconsistentStateException e)
{
logger.Warn(888, $"reload from storage triggered by e-tag mismatch {e}");
problemFlag = TransactionalStatus.StorageConflict;
}
catch (Exception e)
{
logger.Warn(888, $"exception in storageWorker", e);
problemFlag = TransactionalStatus.UnknownException;
} finally
{
if (problemFlag == TransactionalStatus.Ok)
{
this.failCounter = 0;
}
else
{
// after exceptions, we try again, but with limits
if (++failCounter < 10)
{
await Task.Delay(100);
// this restarts the worker, which sees the problem flag and recovers.
storageWorker.Notify();
}
else
{
// bail out
logger.Warn(999, $"storageWorker is bailing out");
}
}
}
}
private async Task StorageWork()
{
try
{
// count committable entries at the bottom of the commit queue
int committableEntries = 0;
while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
{
committableEntries++;
}
// process all committable entries, assembling a storage batch
if (committableEntries > 0)
{
// process all committable entries, adding storage events to the storage batch
CollectEventsForBatch(committableEntries);
if (logger.IsEnabled(LogLevel.Debug))
{
var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
}
}
else
{
// send or re-send messages and detect timeouts
await CheckProgressOfCommitQueue();
}
// store the current storage batch, if it is not empty
StorageBatch <TState> batchBeingSentToStorage = null;
if (this.storageBatch.BatchSize > 0)
{
// get the next batch in place so it can be filled while we store the old one
batchBeingSentToStorage = this.storageBatch;
this.storageBatch = new StorageBatch <TState>(batchBeingSentToStorage);
try
{
if (await batchBeingSentToStorage.CheckStorePreConditions())
{
// perform the actual store, and record the e-tag
this.storageBatch.ETag = await batchBeingSentToStorage.Store(storage);
failCounter = 0;
}
else
{
logger.LogWarning("Store pre conditions not met.");
await AbortAndRestore(TransactionalStatus.CommitFailure);
return;
}
}
catch (InconsistentStateException e)
{
logger.LogWarning(888, e, "Reload from storage triggered by e-tag mismatch.");
await AbortAndRestore(TransactionalStatus.StorageConflict, true);
return;
}
catch (Exception e)
{
logger.Warn(888, $"Storage exception in storageWorker.", e);
await AbortAndRestore(TransactionalStatus.UnknownException);
return;
}
}
if (committableEntries > 0)
{
// update stable state
var lastCommittedEntry = commitQueue[committableEntries - 1];
this.stableState = lastCommittedEntry.State;
this.stableSequenceNumber = lastCommittedEntry.SequenceNumber;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"Stable state version: {this.stableSequenceNumber}");
}
// remove committed entries from commit queue
commitQueue.RemoveFromFront(committableEntries);
storageWorker.Notify(); // we have to re-check for work
}
if (batchBeingSentToStorage != null)
{
batchBeingSentToStorage.RunFollowUpActions();
storageWorker.Notify(); // we have to re-check for work
}
}
catch (Exception e)
{
logger.LogWarning(888, e, "Exception in storageWorker. Retry {FailCounter}", failCounter);
await AbortAndRestore(TransactionalStatus.UnknownException);
}
}