public static OrleansTransactionException ConvertToUserException(this TransactionalStatus status, string transactionId, Exception exception)
{
switch (status)
{
case TransactionalStatus.PrepareTimeout:
return(new OrleansTransactionPrepareTimeoutException(transactionId, exception));
case TransactionalStatus.CascadingAbort:
return(new OrleansCascadingAbortException(transactionId, exception));
case TransactionalStatus.BrokenLock:
return(new OrleansBrokenTransactionLockException(transactionId, "before prepare", exception));
case TransactionalStatus.LockValidationFailed:
return(new OrleansBrokenTransactionLockException(transactionId, "when validating accesses during prepare", exception));
case TransactionalStatus.ParticipantResponseTimeout:
return(new OrleansTransactionTransientFailureException(transactionId, $"transaction agent timed out waiting for read-only transaction participant responses ({status})", exception));
case TransactionalStatus.TMResponseTimeout:
return(new OrleansTransactionInDoubtException(transactionId, $"transaction agent timed out waiting for read-only transaction participant responses ({status})", exception));
case TransactionalStatus.CommitFailure:
return(new OrleansTransactionAbortedException(transactionId, $"Unable to commit transaction ({status})", exception));
default:
return(new OrleansTransactionInDoubtException(transactionId, $"failure during transaction commit, status={status}", exception));
}
}
private async Task <TransactionalStatus> CommitReadWriteTransaction(TransactionInfo transactionInfo, List <ParticipantId> writeResources)
{
ParticipantId manager = SelectManager(transactionInfo, writeResources);
Dictionary <ParticipantId, AccessCounter> participants = transactionInfo.Participants;
foreach (var p in participants
.SelectResources()
.Where(kvp => !kvp.Key.Equals(manager)))
{
// one-way prepare message
p.Key.Reference.AsReference <ITransactionalResourceExtension>()
.Prepare(p.Key.Name, transactionInfo.TransactionId, p.Value, transactionInfo.TimeStamp, manager)
.Ignore();
}
try
{
// wait for the TM to commit the transaction
TransactionalStatus status = await manager.Reference.AsReference <ITransactionManagerExtension>()
.PrepareAndCommit(manager.Name, transactionInfo.TransactionId, participants[manager], transactionInfo.TimeStamp, writeResources, participants.Count);
if (status != TransactionalStatus.Ok)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} fail {transactionInfo.TransactionId} TM response status={status}");
}
// notify participants
if (status.DefinitelyAborted())
{
await Task.WhenAll(writeResources
.Where(p => !p.Equals(manager))
.Select(p => p.Reference.AsReference <ITransactionalResourceExtension>()
.Cancel(p.Name, transactionInfo.TransactionId, transactionInfo.TimeStamp, status)));
}
return(status);
}
}
catch (TimeoutException)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} timeout {transactionInfo.TransactionId} TM response");
}
return(TransactionalStatus.TMResponseTimeout);
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"{stopwatch.Elapsed.TotalMilliseconds:f2} finish {transactionInfo.TransactionId}");
}
return(TransactionalStatus.Ok);
}
private void AbortCommits(TransactionalStatus status, int from = 0)
{
// emtpy the back of the commit queue, starting at specified position
for (int i = from; i < commitQueue.Count; i++)
{
NotifyOfAbort(commitQueue[i], i == from ? status : TransactionalStatus.CascadingAbort);
}
commitQueue.RemoveFromBack(commitQueue.Count - from);
this.RWLock.AbortExecutingTransactions();
}
public void NotifyOfPrepared(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
var pos = commitQueue.Find(transactionId, timeStamp);
if (pos != -1)
{
var localEntry = commitQueue[pos];
if (localEntry.Role != CommitRole.LocalCommit)
{
logger.Error(666, $"transaction abort due to internal error in {nameof(NotifyOfPrepared)}: Wrong commit type");
throw new InvalidOperationException($"Wrong commit type: {localEntry.Role}");
}
if (status == TransactionalStatus.Ok)
{
localEntry.WaitCount--;
storageWorker.Notify();
}
else
{
AbortCommits(status, pos);
this.RWLock.Notify();
}
}
else
{
// this message has arrived ahead of the commit request - we need to remember it
if (this.unprocessedPreparedMessages == null)
{
this.unprocessedPreparedMessages = new Dictionary <DateTime, PMessages>();
}
PMessages info;
if (!this.unprocessedPreparedMessages.TryGetValue(timeStamp, out info))
{
this.unprocessedPreparedMessages[timeStamp] = info = new PMessages();
}
if (status == TransactionalStatus.Ok)
{
info.Count++;
}
else
{
info.Status = status;
}
// TODO fix memory leak if corresponding commit messages never arrive
}
}
public async Task Cancel(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
this.logger.Info($"Grain {this.context.GrainInstance} canceling transaction {transactionId}");
await this.tResource.Cancel(transactionId, timeStamp, status);
if (this.deactivationPhaseReference.DeactivationPhase == TransactionDeactivationPhase.AfterCancel)
{
this.grainRuntime.DeactivateOnIdle((context.GrainInstance));
this.deactivationPhaseReference.DeactivationPhase = TransactionDeactivationPhase.None;
this.logger.Info($"Grain {this.context.GrainInstance} deactivating after transaction {transactionId} cancel");
}
}
public async Task Cancel(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
this.logger.Info($"Grain {this.context.GrainInstance} canceling transaction {transactionId}");
await this.tResource.Cancel(transactionId, timeStamp, status);
if (this.faultInjectionControl.FaultInjectionPhase == TransactionFaultInjectPhase.AfterCancel &&
this.faultInjectionControl.FaultInjectionType == FaultInjectionType.Deactivation)
{
this.grainRuntime.DeactivateOnIdle((context.GrainInstance));
this.logger.Info($"Grain {this.context.GrainInstance} deactivating after transaction {transactionId} cancel");
}
this.faultInjectionControl.Reset();
}
public async Task NotifyOfPrepared(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
var pos = commitQueue.Find(transactionId, timeStamp);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace("NotifyOfPrepared - TransactionId:{TransactionId} Timestamp:{Timestamp}, TransactionalStatus{TransactionalStatus}", transactionId, timeStamp, status);
}
if (pos != -1)
{
var localEntry = commitQueue[pos];
if (localEntry.Role != CommitRole.LocalCommit)
{
logger.LogError($"Transaction abort due to internal error in {nameof(NotifyOfPrepared)}: Wrong commit type");
throw new InvalidOperationException($"Wrong commit type: {localEntry.Role}");
}
if (status == TransactionalStatus.Ok)
{
localEntry.WaitCount--;
storageWorker.Notify();
}
else
{
await AbortCommits(status, pos);
this.RWLock.Notify();
}
}
else
{
// this message has arrived ahead of the commit request - we need to remember it
if (!this.unprocessedPreparedMessages.TryGetValue(timeStamp, out PreparedMessages info))
{
this.unprocessedPreparedMessages[timeStamp] = info = new PreparedMessages(status);
}
if (status == TransactionalStatus.Ok)
{
info.Count++;
}
else
{
info.Status = status;
}
// TODO fix memory leak if corresponding commit messages never arrive
}
}
private async Task AbortCommits(TransactionalStatus status, int from = 0)
{
List <Task> pending = new List <Task>();
// emtpy the back of the commit queue, starting at specified position
for (int i = from; i < commitQueue.Count; i++)
{
pending.Add(NotifyOfAbort(commitQueue[i], i == from ? status : TransactionalStatus.CascadingAbort));
}
commitQueue.RemoveFromBack(commitQueue.Count - from);
pending.Add(this.RWLock.AbortExecutingTransactions());
await Task.WhenAll(pending);
}
public static bool DefinitelyAborted(this TransactionalStatus status)
{
switch (status)
{
case TransactionalStatus.PrepareTimeout:
case TransactionalStatus.CascadingAbort:
case TransactionalStatus.BrokenLock:
case TransactionalStatus.LockValidationFailed:
case TransactionalStatus.ParticipantResponseTimeout:
return(true);
default:
return(false);
}
}
public async Task <TransactionalStatus> Commit(ITransactionInfo info)
{
var transactionInfo = (TransactionInfo)info;
transactionInfo.TimeStamp = this.clock.MergeUtcNow(transactionInfo.TimeStamp);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"{stopwatch.Elapsed.TotalMilliseconds:f2} prepare {transactionInfo}");
}
List <ParticipantId> writeParticipants = null;
foreach (var p in transactionInfo.Participants)
{
if (p.Value.Writes > 0)
{
if (writeParticipants == null)
{
writeParticipants = new List <ParticipantId>();
}
writeParticipants.Add(p.Key);
}
}
try
{
TransactionalStatus status = (writeParticipants == null)
? await CommitReadOnlyTransaction(transactionInfo)
: await CommitReadWriteTransaction(transactionInfo, writeParticipants);
if (status == TransactionalStatus.Ok)
{
this.statistics.TrackTransactionSucceeded();
}
else
{
this.statistics.TrackTransactionFailed();
}
return(status);
}
catch (Exception)
{
this.statistics.TrackTransactionFailed();
throw;
}
}
public void NotifyOfCancel(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
// find in queue
var pos = commitQueue.Find(transactionId, timeStamp);
if (pos == -1)
{
return;
}
this.storageBatch.Cancel(commitQueue[pos].SequenceNumber);
AbortCommits(status, pos);
storageWorker.Notify();
this.RWLock.Notify();
}
public async Task <TransactionalStatus> Resolve(ITransactionInfo info)
{
var transactionInfo = (TransactionInfo)info;
transactionInfo.TimeStamp = this.clock.MergeUtcNow(transactionInfo.TimeStamp);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"{stopwatch.Elapsed.TotalMilliseconds:f2} prepare {transactionInfo}");
}
if (transactionInfo.Participants.Count == 0)
{
this.statistics.TrackTransactionSucceeded();
return(TransactionalStatus.Ok);
}
List <ParticipantId> writeParticipants = null;
List <KeyValuePair <ParticipantId, AccessCounter> > resources = null;
KeyValuePair <ParticipantId, AccessCounter>? manager;
CollateParticipants(transactionInfo.Participants, out writeParticipants, out resources, out manager);
try
{
TransactionalStatus status = (writeParticipants == null)
? await CommitReadOnlyTransaction(transactionInfo, resources)
: await CommitReadWriteTransaction(transactionInfo, writeParticipants, resources, manager.Value);
if (status == TransactionalStatus.Ok)
{
this.statistics.TrackTransactionSucceeded();
}
else
{
this.statistics.TrackTransactionFailed();
}
return(status);
}
catch (Exception)
{
this.statistics.TrackTransactionFailed();
throw;
}
}
public Task Cancel(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
// find in queue
var pos = commitQueue.Find(transactionId, timeStamp);
if (pos == -1)
{
return(Task.CompletedTask); // must have already been cancelled
}
storageBatch.Cancel(commitQueue[pos].SequenceNumber);
storageWorker.Notify();
AbortCommits(status, pos);
lockWorker.Notify();
return(Task.CompletedTask);
}
public static OrleansTransactionException ConvertToUserException(this TransactionalStatus status, string TransactionId)
{
switch (status)
{
case TransactionalStatus.PrepareTimeout:
return(new OrleansTransactionPrepareTimeoutException(TransactionId));
case TransactionalStatus.CascadingAbort:
return(new OrleansCascadingAbortException(TransactionId));
case TransactionalStatus.BrokenLock:
return(new OrleansBrokenTransactionLockException(TransactionId, "before prepare"));
case TransactionalStatus.LockValidationFailed:
return(new OrleansBrokenTransactionLockException(TransactionId, "when validating accesses during prepare"));
default:
return(new OrleansTransactionInDoubtException(TransactionId, $"failure during transaction commit, status={status}"));
}
}
private async Task Bail(TransactionalStatus status, bool force = false)
{
await RWLock.AbortExecutingTransactions();
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
await NotifyOfAbort(entry, status);
}
commitQueue.Clear();
this.RWLock.AbortQueuedTransactions();
if (++failCounter >= 10 || force)
{
logger.Debug("StorageWorker triggering grain Deactivation");
this.deactivate();
}
else
{
await this.Restore();
}
}
public void NotifyOfCancel(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("{MethodName}. TransactionId: {TransactionId}, TimeStamp: {TimeStamp} Status: {TransactionalStatus}", nameof(NotifyOfCancel), transactionId, timeStamp, status);
}
// find in queue
var pos = commitQueue.Find(transactionId, timeStamp);
if (pos == -1)
{
return;
}
this.storageBatch.Cancel(commitQueue[pos].SequenceNumber);
AbortCommits(status, pos);
storageWorker.Notify();
this.RWLock.Notify();
}
private async Task Bail(TransactionalStatus status, bool force = false)
{
List <Task> pending = new List <Task>();
pending.Add(RWLock.AbortExecutingTransactions());
this.RWLock.AbortQueuedTransactions();
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
pending.Add(NotifyOfAbort(entry, status));
}
commitQueue.Clear();
await Task.WhenAll(pending);
if (++failCounter >= 10 || force)
{
logger.Debug("StorageWorker triggering grain Deactivation");
this.deactivate();
}
await this.Restore();
}
private async Task <TransactionalStatus> CommitReadWriteTransaction(TransactionInfo transactionInfo, List <ParticipantId> writeResources, List <KeyValuePair <ParticipantId, AccessCounter> > resources, KeyValuePair <ParticipantId, AccessCounter> manager)
{
TransactionalStatus status = TransactionalStatus.Ok;
try
{
foreach (var p in resources)
{
if (p.Key.Equals(manager.Key))
{
continue;
}
// one-way prepare message
p.Key.Reference.AsReference <ITransactionalResourceExtension>()
.Prepare(p.Key.Name, transactionInfo.TransactionId, p.Value, transactionInfo.TimeStamp, manager.Key)
.Ignore();
}
// wait for the TM to commit the transaction
status = await manager.Key.Reference.AsReference <ITransactionManagerExtension>()
.PrepareAndCommit(manager.Key.Name, transactionInfo.TransactionId, manager.Value, transactionInfo.TimeStamp, writeResources, resources.Count);
}
catch (TimeoutException)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} timeout {transactionInfo.TransactionId} on CommitReadWriteTransaction");
}
status = TransactionalStatus.TMResponseTimeout;
}
catch (Exception ex)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} failure {transactionInfo.TransactionId} CommitReadWriteTransaction");
}
this.logger.LogWarning(ex, "Unknown error while commiting transaction {TransactionId}", transactionInfo.TransactionId);
status = TransactionalStatus.PresumedAbort;
}
if (status != TransactionalStatus.Ok)
{
try
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} failed {transactionInfo.TransactionId} with status={status}");
}
// notify participants
if (status.DefinitelyAborted())
{
await Task.WhenAll(writeResources
.Where(p => !p.Equals(manager.Key))
.Select(p => p.Reference.AsReference <ITransactionalResourceExtension>()
.Cancel(p.Name, transactionInfo.TransactionId, transactionInfo.TimeStamp, status)));
}
}
catch (Exception ex)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} failure aborting {transactionInfo.TransactionId} CommitReadWriteTransaction");
}
this.logger.LogWarning(ex, "Failed to abort transaction {TransactionId}", transactionInfo.TransactionId);
}
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"{stopwatch.Elapsed.TotalMilliseconds:f2} finish {transactionInfo.TransactionId}");
}
return(status);
}
public Task Cancel(string resourceId, Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
return(GetResource(resourceId).Cancel(transactionId, timeStamp, status));
}
public Task Prepared(Guid transactionId, DateTime timeStamp, ParticipantId resource, TransactionalStatus status)
{
this.queue.NotifyOfPrepared(transactionId, timeStamp, status);
return(Task.CompletedTask);
}
private async Task StorageWork()
{
try
{
if (problemFlag != TransactionalStatus.Ok)
{
RWLock.AbortExecutingTransactions();
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
NotifyOfAbort(entry, problemFlag);
}
commitQueue.Clear();
if (problemFlag == TransactionalStatus.StorageConflict)
{
logger.Debug("deactivating after storage conflict");
this.deactivate();
this.RWLock.AbortQueuedTransactions();
}
else
{
logger.Debug($"restoring state after status={problemFlag}");
// recover, clear storageFlag, then allow next queued transaction(s) to enter lock
await NotifyOfRestore();
}
}
else
{
// count committable entries at the bottom of the commit queue
int committableEntries = 0;
while (committableEntries < commitQueue.Count && commitQueue[committableEntries].ReadyToCommit)
{
committableEntries++;
}
// process all committable entries, assembling a storage batch
if (committableEntries > 0)
{
// process all committable entries, adding storage events to the storage batch
CollectEventsForBatch(committableEntries);
if (problemFlag != TransactionalStatus.Ok)
{
return;
}
if (logger.IsEnabled(LogLevel.Debug))
{
var r = commitQueue.Count > committableEntries ? commitQueue[committableEntries].ToString() : "";
logger.Debug($"batchcommit={committableEntries} leave={commitQueue.Count - committableEntries} {r}");
}
}
else
{
// send or re-send messages and detect timeouts
CheckProgressOfCommitQueue();
}
// store the current storage batch, if it is not empty
StorageBatch <TState> batchBeingSentToStorage = null;
if (this.storageBatch.BatchSize > 0)
{
// get the next batch in place so it can be filled while we store the old one
batchBeingSentToStorage = this.storageBatch;
this.storageBatch = new StorageBatch <TState>(batchBeingSentToStorage);
// perform the actual store, and record the e-tag
this.storageBatch.ETag = await batchBeingSentToStorage.Store(storage);
}
if (committableEntries > 0)
{
// update stable state
var lastCommittedEntry = commitQueue[committableEntries - 1];
this.stableState = lastCommittedEntry.State;
this.stableSequenceNumber = lastCommittedEntry.SequenceNumber;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"Stable state version: {this.stableSequenceNumber}");
}
// remove committed entries from commit queue
commitQueue.RemoveFromFront(committableEntries);
storageWorker.Notify(); // we have to re-check for work
}
if (batchBeingSentToStorage != null)
{
batchBeingSentToStorage.RunFollowUpActions();
storageWorker.Notify(); // we have to re-check for work
}
}
}
catch (InconsistentStateException e)
{
logger.Warn(888, $"reload from storage triggered by e-tag mismatch {e}");
problemFlag = TransactionalStatus.StorageConflict;
}
catch (Exception e)
{
logger.Warn(888, $"exception in storageWorker", e);
problemFlag = TransactionalStatus.UnknownException;
} finally
{
if (problemFlag == TransactionalStatus.Ok)
{
this.failCounter = 0;
}
else
{
// after exceptions, we try again, but with limits
if (++failCounter < 10)
{
await Task.Delay(100);
// this restarts the worker, which sees the problem flag and recovers.
storageWorker.Notify();
}
else
{
// bail out
logger.Warn(999, $"storageWorker is bailing out");
}
}
}
}
private async Task Restore()
{
TransactionalStorageLoadResponse <TState> loadresponse = await storage.Load();
this.storageBatch = new StorageBatch <TState>(loadresponse, this.serializerSettings);
this.stableState = loadresponse.CommittedState;
this.stableSequenceNumber = loadresponse.CommittedSequenceId;
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"Load v{this.stableSequenceNumber} {loadresponse.PendingStates.Count}p {storageBatch.MetaData.CommitRecords.Count}c");
}
// ensure clock is consistent with loaded state
this.Clock.Merge(storageBatch.MetaData.TimeStamp);
// resume prepared transactions (not TM)
foreach (var pr in loadresponse.PendingStates.OrderBy(ps => ps.TimeStamp))
{
if (pr.SequenceId > loadresponse.CommittedSequenceId && pr.TransactionManager != null)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"recover two-phase-commit {pr.TransactionId}");
}
ParticipantId tm = JsonConvert.DeserializeObject <ParticipantId>(pr.TransactionManager, this.serializerSettings);
commitQueue.Add(new TransactionRecord <TState>()
{
Role = CommitRole.RemoteCommit,
TransactionId = Guid.Parse(pr.TransactionId),
Timestamp = pr.TimeStamp,
State = pr.State,
SequenceNumber = pr.SequenceId,
TransactionManager = tm,
PrepareIsPersisted = true,
LastSent = default(DateTime),
ConfirmationResponsePromise = null,
NumberWrites = 1 // was a writing transaction
});
this.stableSequenceNumber = pr.SequenceId;
}
}
// resume committed transactions (on TM)
foreach (var kvp in storageBatch.MetaData.CommitRecords)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"recover commit confirmation {kvp.Key}");
}
confirmationTasks.Add(kvp.Key, new TransactionRecord <TState>()
{
Role = CommitRole.LocalCommit,
TransactionId = kvp.Key,
Timestamp = kvp.Value.Timestamp,
WriteParticipants = kvp.Value.WriteParticipants
});
}
// clear the problem flag
problemFlag = TransactionalStatus.Ok;
// check for work
this.confirmationWorker.Notify();
this.storageWorker.Notify();
this.RWLock.Notify();
}
public PreparedMessages(TransactionalStatus status)
{
this.Status = status;
}
public void NotifyOfAbort(TransactionRecord <TState> entry, TransactionalStatus status)
{
switch (entry.Role)
{
case CommitRole.NotYetDetermined:
{
// cannot notify anyone. TA will detect broken lock during prepare.
break;
}
case CommitRole.RemoteCommit:
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"aborting RemoteCommitEntry {entry.Timestamp:o} status={status}");
}
entry.ConfirmationResponsePromise?.TrySetException(new OrleansException($"Confirm failed: Status {status}"));
if (entry.LastSent.HasValue)
{
return; // cannot abort anymore if we already sent prepare-ok message
}
entry.TransactionManager.Reference.AsReference <ITransactionManagerExtension>()
.Prepared(entry.TransactionManager.Name, entry.TransactionId, entry.Timestamp, resource, status)
.Ignore();
break;
}
case CommitRole.LocalCommit:
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"aborting LocalCommitEntry {entry.Timestamp:o} status={status}");
}
// reply to transaction agent
entry.PromiseForTA.TrySetResult(status);
// tell remote participants
foreach (var p in entry.WriteParticipants)
{
if (!p.Equals(resource))
{
p.Reference.AsReference <ITransactionalResourceExtension>()
.Cancel(p.Name, entry.TransactionId, entry.Timestamp, status)
.Ignore();
}
}
break;
}
case CommitRole.ReadOnly:
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"aborting ReadEntry {entry.Timestamp:o} status={status}");
}
// reply to transaction agent
entry.PromiseForTA.TrySetResult(status);
break;
}
default:
{
logger.LogError(777, "internal error: impossible case {CommitRole}", entry.Role);
throw new NotSupportedException($"{entry.Role} is not a supported CommitRole.");
}
}
}
private async Task <TransactionalStatus> CommitReadOnlyTransaction(TransactionInfo transactionInfo, List <KeyValuePair <ParticipantId, AccessCounter> > resources)
{
TransactionalStatus status = TransactionalStatus.Ok;
var tasks = new List <Task <TransactionalStatus> >();
try
{
foreach (KeyValuePair <ParticipantId, AccessCounter> resource in resources)
{
tasks.Add(resource.Key.Reference.AsReference <ITransactionalResourceExtension>()
.CommitReadOnly(resource.Key.Name, transactionInfo.TransactionId, resource.Value, transactionInfo.TimeStamp));
}
// wait for all responses
TransactionalStatus[] results = await Task.WhenAll(tasks);
// examine the return status
foreach (var s in results)
{
if (s != TransactionalStatus.Ok)
{
status = s;
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} fail {transactionInfo.TransactionId} prepare response status={status}");
}
break;
}
}
}
catch (TimeoutException)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} timeout {transactionInfo.TransactionId} on CommitReadOnly");
}
status = TransactionalStatus.ParticipantResponseTimeout;
}
catch (Exception ex)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} failure {transactionInfo.TransactionId} CommitReadOnly");
}
this.logger.LogWarning(ex, "Unknown error while commiting readonly transaction {TransactionId}", transactionInfo.TransactionId);
status = TransactionalStatus.PresumedAbort;
}
if (status != TransactionalStatus.Ok)
{
try
{
await Task.WhenAll(resources.Select(r => r.Key.Reference.AsReference <ITransactionalResourceExtension>()
.Abort(r.Key.Name, transactionInfo.TransactionId)));
}
catch (Exception ex)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"{stopwatch.Elapsed.TotalMilliseconds:f2} failure aborting {transactionInfo.TransactionId} CommitReadOnly");
}
this.logger.LogWarning(ex, "Failed to abort readonly transaction {TransactionId}", transactionInfo.TransactionId);
}
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"{stopwatch.Elapsed.TotalMilliseconds:f2} finish (reads only) {transactionInfo.TransactionId}");
}
return(status);
}
private void NotifyOfAbort(TransactionRecord <TState> entry, TransactionalStatus status)
{
switch (entry.Role)
{
case CommitRole.NotYetDetermined:
{
// cannot notify anyone. TA will detect broken lock during prepare.
break;
}
case CommitRole.RemoteCommit:
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"aborting RemoteCommitEntry {entry.Timestamp:o} status={status}");
}
if (entry.LastSent.HasValue)
{
return; // cannot abort anymore if we already sent prepare-ok message
}
entry.TransactionManager.Prepared(entry.TransactionId, entry.Timestamp, thisParticipant, status).Ignore();
break;
}
case CommitRole.LocalCommit:
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"aborting LocalCommitEntry {entry.Timestamp:o} status={status}");
}
// reply to transaction agent
entry.PromiseForTA.TrySetResult(status);
// tell remote participants
foreach (var p in entry.WriteParticipants)
{
if (p != thisParticipant)
{
p.Cancel(entry.TransactionId, entry.Timestamp, status).Ignore();
}
}
break;
}
case CommitRole.ReadOnly:
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace($"aborting ReadEntry {entry.Timestamp:o} status={status}");
}
// reply to transaction agent
entry.PromiseForTA.TrySetResult(status);
break;
}
default:
{
logger.LogError(777, "internal error: impossible case {CommitRole}", entry.Role);
throw new NotSupportedException($"{entry.Role} is not a supported CommitRole.");
}
}
}
/// <summary>
/// called on activation, and when recovering from storage conflicts or other exceptions.
/// </summary>
private async Task Restore()
{
// start the load
var loadtask = this.storage.Load();
// abort active transactions, without waking up waiters just yet
AbortExecutingTransactions("due to restore");
// abort all entries in the commit queue
foreach (var entry in commitQueue.Elements)
{
NotifyOfAbort(entry, problemFlag);
}
commitQueue.Clear();
var loadresponse = await loadtask;
storageBatch = new StorageBatch <TState>(loadresponse);
stableState = loadresponse.CommittedState;
stableSequenceNumber = loadresponse.CommittedSequenceId;
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"Load v{stableSequenceNumber} {loadresponse.PendingStates.Count}p {storageBatch.MetaData.CommitRecords.Count}c");
}
// ensure clock is consistent with loaded state
MergeClock(storageBatch.MetaData.TimeStamp);
// resume prepared transactions (not TM)
foreach (var pr in loadresponse.PendingStates.OrderBy(ps => ps.TimeStamp))
{
if (pr.SequenceId > stableSequenceNumber && pr.TransactionManager != null)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"recover two-phase-commit {pr.TransactionId}");
}
var tm = (pr.TransactionManager == null) ? null :
(ITransactionParticipant)JsonConvert.DeserializeObject <ITransactionParticipant>(pr.TransactionManager, MetaData.SerializerSettings);
commitQueue.Add(new TransactionRecord <TState>()
{
Role = CommitRole.RemoteCommit,
TransactionId = Guid.Parse(pr.TransactionId),
Timestamp = pr.TimeStamp,
State = pr.State,
TransactionManager = tm,
PrepareIsPersisted = true,
LastSent = default(DateTime),
ConfirmationResponsePromise = null
});
}
}
// resume committed transactions (on TM)
foreach (var kvp in storageBatch.MetaData.CommitRecords)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug($"recover commit confirmation {kvp.Key}");
}
confirmationTasks.Add(kvp.Key, new TransactionRecord <TState>()
{
Role = CommitRole.LocalCommit,
TransactionId = kvp.Key,
Timestamp = kvp.Value.Timestamp,
WriteParticipants = kvp.Value.WriteParticipants
});
}
// clear the problem flag
problemFlag = TransactionalStatus.Ok;
// check for work
confirmationWorker.Notify();
storageWorker.Notify();
lockWorker.Notify();
}
public Task Prepared(Guid transactionId, DateTime timeStamp, ITransactionParticipant participant, TransactionalStatus status)
{
return(extension.Prepared(resourceId, transactionId, timeStamp, participant, status));
}
public Task Prepared(string resourceId, Guid transactionId, DateTime timestamp, ParticipantId resource, TransactionalStatus status)
{
return(GetManager(resourceId).Prepared(transactionId, timestamp, resource, status));
}
public Task Cancel(Guid transactionId, DateTime timeStamp, TransactionalStatus status)
{
return(extension.Cancel(resourceId, transactionId, timeStamp, status));
}
