public StorageBatch(StorageBatch <TState> previous)
{
MetaData = previous.MetaData;
confirmUpTo = previous.confirmUpTo;
cancelAbove = previous.cancelAbove;
cancelAboveStart = cancelAbove;
}
public StorageBatch(StorageBatch <TState> previous)
{
this.serializerSettings = previous.serializerSettings;
MetaData = previous.MetaData;
confirmUpTo = previous.confirmUpTo;
cancelAbove = previous.cancelAbove;
cancelAboveStart = cancelAbove;
}
/// <summary>
/// called on activation, and when recovering from storage conflicts or other exceptions.
/// </summary>
private async Task Restore()
{
// start the load
var loadtask = this.storage.Load();
// abort active transactions, without waking up waiters just yet
AbortExecutingTransactions("due to restore");
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
NotifyOfAbort(entry, problemFlag);
}
commitQueue.Clear();
var loadresponse = await loadtask;
storageBatch = new StorageBatch <TState>(loadresponse);
stableState = loadresponse.CommittedState;
stableSequenceNumber = loadresponse.CommittedSequenceId;
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"Load v{stableSequenceNumber} {loadresponse.PendingStates.Count}p {storageBatch.MetaData.CommitRecords.Count}c");
}
// ensure clock is consistent with loaded state
MergeClock(storageBatch.MetaData.TimeStamp);
// resume prepared transactions (not TM)
foreach (var pr in loadresponse.PendingStates.OrderBy(ps => ps.TimeStamp))
{
if (pr.SequenceId > stableSequenceNumber && pr.TransactionManager != null)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"recover two-phase-commit {pr.TransactionId}");
}
var tm = (pr.TransactionManager == null) ? null :
(ITransactionParticipant)JsonConvert.DeserializeObject <ITransactionParticipant>(pr.TransactionManager, MetaData.SerializerSettings);
commitQueue.Add(new TransactionRecord <TState>()
{
Role = CommitRole.RemoteCommit,
TransactionId = Guid.Parse(pr.TransactionId),
Timestamp = pr.TimeStamp,
State = pr.State,
TransactionManager = tm,
PrepareIsPersisted = true,
LastSent = default(DateTime),
ConfirmationResponsePromise = null
});
}
}
// resume committed transactions (on TM)
foreach (var kvp in storageBatch.MetaData.CommitRecords)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"recover commit confirmation {kvp.Key}");
}
confirmationTasks.Add(kvp.Key, new TransactionRecord <TState>()
{
Role = CommitRole.LocalCommit,
TransactionId = kvp.Key,
Timestamp = kvp.Value.Timestamp,
WriteParticipants = kvp.Value.WriteParticipants
});
}
// clear the problem flag
problemFlag = TransactionalStatus.Ok;
// check for work
confirmationWorker.Notify();
storageWorker.Notify();
lockWorker.Notify();
}
private async Task StorageWork()
{
try
{
if (problemFlag != TransactionalStatus.Ok)
{
logger.Debug($"restoring state after status={problemFlag}");
await Restore();
}
else
{
// count committable entries at the bottom of the commit queue
int committableEntries = 0;
while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
{
committableEntries++;
}
// process all committable entries, assembling a storage batch
if (committableEntries > 0)
{
// process all committable entries, adding storage events to the storage batch
CollectEventsForBatch(committableEntries);
if (logger.IsEnabled(LogLevel.Debug))
{
var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
}
}
else
{
// send or re-send messages and detect timeouts
CheckProgressOfCommitQueue();
}
// store the current storage batch, if it is not empty
if (storageBatch.BatchSize > 0)
{
var thisBatch = storageBatch;
var nextBatch = new StorageBatch <TState>(thisBatch);
// get the next batch in place so it can be filled while we store the old one
storageBatch = nextBatch;
nextBatch.ETag = await thisBatch.Store(storage);
if (committableEntries > 0)
{
// update stable state
var lastCommittedEntry = commitQueue[committableEntries - 1];
stableState = lastCommittedEntry.State;
stableSequenceNumber = lastCommittedEntry.SequenceNumber;
// remove committed entries from commit queue
commitQueue.RemoveFromFront(committableEntries);
}
// run follow-up actions
thisBatch.RunFollowUpActions();
storageWorker.Notify(); // re-check for work
}
}
return;
}
catch (InconsistentStateException e)
{
logger.Warn(888, $"reload from storage triggered by e-tag mismatch {e}");
problemFlag = TransactionalStatus.StorageConflict;
}
catch (Exception e)
{
logger.Warn(888, $"exception in storageWorker", e);
problemFlag = TransactionalStatus.UnknownException;
}
// after exceptions, we try again, but with limits
if (++failCounter < 10)
{
await Task.Delay(100);
// this restarts the worker, which sees the problem flag and recovers.
storageWorker.Notify();
}
else
{
// bail out
logger.Warn(999, $"storageWorker is bailing out");
}
}
private async Task StorageWork()
{
try
{
if (problemFlag != TransactionalStatus.Ok)
{
// abort active transactions
AbortExecutingTransactions($"due to status={problemFlag}");
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
NotifyOfAbort(entry, problemFlag);
}
commitQueue.Clear();
if (problemFlag == TransactionalStatus.StorageConflict)
{
logger.Debug("deactivating after storage conflict");
grainRuntime.DeactivateOnIdle(context.GrainInstance);
AbortQueuedTransactions();
}
else
{
logger.Debug($"restoring state after status={problemFlag}");
// recover, clear storageFlag, then allow next queued transaction(s) to enter lock
await Restore();
}
}
else
{
// count committable entries at the bottom of the commit queue
int committableEntries = 0;
while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
{
committableEntries++;
}
// process all committable entries, assembling a storage batch
if (committableEntries > 0)
{
// process all committable entries, adding storage events to the storage batch
CollectEventsForBatch(committableEntries);
if (logger.IsEnabled(LogLevel.Debug))
{
var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
}
}
else
{
// send or re-send messages and detect timeouts
CheckProgressOfCommitQueue();
}
// store the current storage batch, if it is not empty
StorageBatch <TState> batchBeingSentToStorage = null;
if (storageBatch.BatchSize > 0)
{
// get the next batch in place so it can be filled while we store the old one
batchBeingSentToStorage = storageBatch;
storageBatch = new StorageBatch <TState>(batchBeingSentToStorage);
// perform the actual store, and record the e-tag
storageBatch.ETag = await batchBeingSentToStorage.Store(storage);
}
if (committableEntries > 0)
{
// update stable state
var lastCommittedEntry = commitQueue[committableEntries - 1];
stableState = lastCommittedEntry.State;
stableSequenceNumber = lastCommittedEntry.SequenceNumber;
// remove committed entries from commit queue
commitQueue.RemoveFromFront(committableEntries);
storageWorker.Notify(); // we have to re-check for work
}
if (batchBeingSentToStorage != null)
{
batchBeingSentToStorage.RunFollowUpActions();
storageWorker.Notify(); // we have to re-check for work
}
}
return;
}
catch (InconsistentStateException e)
{
logger.Warn(888, $"reload from storage triggered by e-tag mismatch {e}");
problemFlag = TransactionalStatus.StorageConflict;
}
catch (Exception e)
{
logger.Warn(888, $"exception in storageWorker", e);
problemFlag = TransactionalStatus.UnknownException;
}
// after exceptions, we try again, but with limits
if (++failCounter < 10)
{
await Task.Delay(100);
// this restarts the worker, which sees the problem flag and recovers.
storageWorker.Notify();
}
else
{
// bail out
logger.Warn(999, $"storageWorker is bailing out");
}
}
public StorageBatch(StorageBatch <TState> previous)
: this(previous.MetaData, previous.ETag, previous.confirmUpTo, previous.cancelAbove)
{
}