public virtual async Task WrongEtags()
{
var stateStorage = await this.stateStorageFactory();
// load first time
var loadresponse = await stateStorage.Load();
// store with wrong e-tag, must fail
try
{
var etag1 = await stateStorage.Store("wrong-etag", loadresponse.Metadata, emptyPendingStates, null, null);
throw new Exception("storage did not catch e-tag mismatch");
}
catch (Exception) { }
// load again
loadresponse = await stateStorage.Load();
loadresponse.Should().NotBeNull();
loadresponse.Metadata.TimeStamp.Should().Be(default(DateTime));
loadresponse.Metadata.CommitRecords.Should().BeEmpty();
loadresponse.ETag.Should().BeNull();
loadresponse.CommittedSequenceId.Should().Be(0);
loadresponse.PendingStates.Should().BeEmpty();
// update timestamp in metadata, then write back with correct e-tag
var now = DateTime.UtcNow;
var cr = MakeCommitRecords(2, 2);
var metadata = new TransactionalStateMetaData()
{
TimeStamp = now, CommitRecords = cr
};
var etag2 = await stateStorage.Store(null, metadata, emptyPendingStates, null, null);
// update timestamp in metadata, then write back with wrong e-tag, must fail
try
{
var now2 = DateTime.UtcNow;
var metadata2 = new TransactionalStateMetaData()
{
TimeStamp = now2, CommitRecords = MakeCommitRecords(3, 3)
};
await stateStorage.Store(null, metadata, emptyPendingStates, null, null);
throw new Exception("storage did not catch e-tag mismatch");
}
catch (Exception) { }
// load again, check content
loadresponse = await stateStorage.Load();
loadresponse.Should().NotBeNull();
loadresponse.Metadata.Should().NotBeNull();
loadresponse.Metadata.TimeStamp.Should().Be(now);
loadresponse.Metadata.CommitRecords.Count.Should().Be(cr.Count);
loadresponse.ETag.Should().Be(etag2);
loadresponse.CommittedSequenceId.Should().Be(0);
loadresponse.PendingStates.Should().BeEmpty();
}
public void JsonConcertCanSerializeMetaData()
{
ITransactionTestGrain testGrain = this.RandomTestGrain(TransactionTestConstants.SingleStateTransactionalGrain);
GrainReference reference = (GrainReference)testGrain;
var metaData = new TransactionalStateMetaData();
metaData.TimeStamp = DateTime.UtcNow;
metaData.CommitRecords.Add(Guid.NewGuid(), new CommitRecord()
{
Timestamp = DateTime.UtcNow,
WriteParticipants = new List <ParticipantId>()
{
new ParticipantId("bob", reference, ParticipantId.Role.Resource | ParticipantId.Role.Manager)
}
});
JsonSerializerSettings serializerSettings = TransactionalStateFactory.GetJsonSerializerSettings(
this.fixture.Client.ServiceProvider.GetService <ITypeResolver>(),
this.grainFactory);
//should be able to serialize it
string jsonMetaData = JsonConvert.SerializeObject(metaData, serializerSettings);
TransactionalStateMetaData deseriliazedMetaData = JsonConvert.DeserializeObject <TransactionalStateMetaData>(jsonMetaData, serializerSettings);
Assert.Equal(metaData.TimeStamp, deseriliazedMetaData.TimeStamp);
}
public StorageBatch(TransactionalStateMetaData metaData, string etag, long confirmUpTo, long cancelAbove)
{
this.MetaData = metaData ?? throw new ArgumentNullException(nameof(metaData));
this.ETag = etag;
this.confirmUpTo = confirmUpTo;
this.cancelAbove = cancelAbove;
this.cancelAboveStart = cancelAbove;
this.followUpActions = new List <Action>();
this.storeConditions = new List <Func <Task <bool> > >();
this.prepares = new SortedDictionary <long, PendingTransactionState <TState> >();
}
public async Task <string> Store(string expectedETag, TransactionalStateMetaData metadata,
List <PendingTransactionState <TState> > statesToPrepare, long?commitUpTo,
long?abortAfter)
{
if (abortAfter.HasValue && _states.Any())
{
while (_states.Count > 0 && _states[_states.Count - 1].SequenceId > abortAfter)
{
var entity = _states[_states.Count - 1];
await RemoveAbortedState(entity).ConfigureAwait(false);
_states.RemoveAt(_states.Count - 1);
}
}
if (statesToPrepare != null)
{
foreach (var s in statesToPrepare)
{
ITransactionStateEntity <TState> existingState = null;
if (FindState(s.SequenceId, out var pos))
{
existingState = _states[pos];
}
var persistedState = await PersistState(s, commitUpTo, existingState).ConfigureAwait(false);
if (existingState == null)
{
_states.Insert(pos, persistedState);
}
else
{
_states[pos] = persistedState;
}
}
}
Metadata = await PersistMetadata(metadata, commitUpTo ?? Metadata.CommittedSequenceId)
.ConfigureAwait(false);
await StoreFinalize(commitUpTo).ConfigureAwait(false);
return(Metadata.ETag);
}
public virtual async Task StoreWithoutChanges()
{
var stateStorage = await this.stateStorageFactory();
// load first time
var loadresponse = await stateStorage.Load();
// store without any changes
var etag1 = await stateStorage.Store(loadresponse.ETag, loadresponse.Metadata, emptyPendingStates, null, null);
// load again
loadresponse = await stateStorage.Load();
loadresponse.Should().NotBeNull();
loadresponse.Metadata.Should().NotBeNull();
loadresponse.Metadata.TimeStamp.Should().Be(default(DateTime));
loadresponse.Metadata.CommitRecords.Should().BeEmpty();
loadresponse.ETag.Should().Be(etag1);
loadresponse.CommittedSequenceId.Should().Be(0);
loadresponse.PendingStates.Should().BeEmpty();
// update metadata, then write back
var now = DateTime.UtcNow;
var cr = MakeCommitRecords(2, 2);
var metadata = new TransactionalStateMetaData()
{
TimeStamp = now, CommitRecords = cr
};
var etag2 = await stateStorage.Store(etag1, metadata, emptyPendingStates, null, null);
// load again, check content
loadresponse = await stateStorage.Load();
loadresponse.Should().NotBeNull();
loadresponse.Metadata.Should().NotBeNull();
loadresponse.Metadata.TimeStamp.Should().Be(now);
loadresponse.Metadata.CommitRecords.Count.Should().Be(cr.Count);
loadresponse.ETag.Should().Be(etag2);
loadresponse.CommittedSequenceId.Should().Be(0);
loadresponse.PendingStates.Should().BeEmpty();
}
public async Task <string> Store(
string expectedETag,
TransactionalStateMetaData metadata,
// a list of transactions to prepare.
List <PendingTransactionState <TState> > statesToPrepare,
// if non-null, commit all pending transaction up to and including this sequence number.
long?commitUpTo,
// if non-null, abort all pending transactions with sequence numbers strictly larger than this one.
long?abortAfter
)
{
faultInjector.BeforeStore();
var result = await this.stateStorage.Store(expectedETag, metadata, statesToPrepare, commitUpTo, abortAfter);
faultInjector.AfterStore();
return(result);
}
protected override Task <ITransactionMetadataEntity> PersistMetadata(TransactionalStateMetaData value,
long commitSequenceId) => ExecuteQuery <ITransactionMetadataEntity>(async db =>
{
var tableName = _options.MetadataTableName;
var newEtag = Guid.NewGuid().ToString();
var serializedValue = JsonConvert.SerializeObject(value, _jsonSettings);
if (Metadata.ETag == null)
{
await db.Query(tableName)
.AsInsert(new[] { "state_id", "committed_sequence_id", "etag", "value" },
new object[]
{
_stateId, commitSequenceId, newEtag, serializedValue
}).FirstOrDefaultAsync().ConfigureAwait(false);
}
else
{
var rowsUpdated = await db.Query(tableName).Where("state_id", _stateId).Where("etag", Metadata.ETag)
.UpdateAsync(new
{
committed_sequence_id = commitSequenceId,
etag = newEtag,
value = serializedValue
}).ConfigureAwait(false);
if (rowsUpdated != 1)
{
throw new InvalidOperationException("Could not update metadata. Possible concurrency issue");
}
}
return(new TransactionMetadata
{
Value = value,
ETag = newEtag,
CommittedSequenceId = commitSequenceId
});
});
public async Task <TransactionalStorageLoadResponse <TState> > Load()
{
try
{
var keyTask = ReadKey();
var statesTask = ReadStates();
key = await keyTask.ConfigureAwait(false);
states = await statesTask.ConfigureAwait(false);
if (string.IsNullOrEmpty(key.ETag.ToString()))
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug($"{partition} Loaded v0, fresh");
}
// first time load
return(new TransactionalStorageLoadResponse <TState>());
}
else
{
TState committedState;
if (this.key.CommittedSequenceId == 0)
{
committedState = new TState();
}
else
{
if (!FindState(this.key.CommittedSequenceId, out var pos))
{
var error = $"Storage state corrupted: no record for committed state v{this.key.CommittedSequenceId}";
logger.LogCritical($"{partition} {error}");
throw new InvalidOperationException(error);
}
committedState = states[pos].Value.GetState <TState>(this.jsonSettings);
}
var PrepareRecordsToRecover = new List <PendingTransactionState <TState> >();
for (int i = 0; i < states.Count; i++)
{
var kvp = states[i];
// pending states for already committed transactions can be ignored
if (kvp.Key <= key.CommittedSequenceId)
{
continue;
}
// upon recovery, local non-committed transactions are considered aborted
if (kvp.Value.TransactionManager == null)
{
break;
}
ParticipantId tm = JsonConvert.DeserializeObject <ParticipantId>(kvp.Value.TransactionManager, this.jsonSettings);
PrepareRecordsToRecover.Add(new PendingTransactionState <TState>()
{
SequenceId = kvp.Key,
State = kvp.Value.GetState <TState>(this.jsonSettings),
TimeStamp = kvp.Value.TransactionTimestamp,
TransactionId = kvp.Value.TransactionId,
TransactionManager = tm
});
}
// clear the state strings... no longer needed, ok to GC now
for (int i = 0; i < states.Count; i++)
{
var entity = states[i].Value;
entity.StateJson = null;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug($"{partition} Loaded v{this.key.CommittedSequenceId} rows={string.Join(",", states.Select(s => s.Key.ToString("x16")))}");
}
TransactionalStateMetaData metadata = JsonConvert.DeserializeObject <TransactionalStateMetaData>(this.key.Metadata, this.jsonSettings);
return(new TransactionalStorageLoadResponse <TState>(this.key.ETag.ToString(), committedState, this.key.CommittedSequenceId, metadata, PrepareRecordsToRecover));
}
}
catch (Exception ex)
{
this.logger.LogError("Transactional state load failed {Exception}.", ex);
throw;
}
}
public async Task <string> Store(string expectedETag, TransactionalStateMetaData metadata, List <PendingTransactionState <TState> > statesToPrepare, long?commitUpTo, long?abortAfter)
{
var keyETag = key.ETag.ToString();
if ((!string.IsNullOrWhiteSpace(keyETag) || !string.IsNullOrWhiteSpace(expectedETag)) && keyETag != expectedETag)
{
throw new ArgumentException(nameof(expectedETag), "Etag does not match");
}
// assemble all storage operations into a single batch
// these operations must commit in sequence, but not necessarily atomically
// so we can split this up if needed
var batchOperation = new BatchOperation(logger, key, table);
// first, clean up aborted records
if (abortAfter.HasValue && states.Count != 0)
{
while (states.Count > 0 && states[states.Count - 1].Key > abortAfter)
{
var entity = states[states.Count - 1].Value;
await batchOperation.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity.Entity, entity.ETag)).ConfigureAwait(false);
key.ETag = batchOperation.KeyETag;
states.RemoveAt(states.Count - 1);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace($"{partition}.{entity.RowKey} Delete {entity.TransactionId}");
}
}
}
// second, persist non-obsolete prepare records
var obsoleteBefore = commitUpTo.HasValue ? commitUpTo.Value : key.CommittedSequenceId;
if (statesToPrepare != null)
{
foreach (var s in statesToPrepare)
{
if (s.SequenceId >= obsoleteBefore)
{
if (FindState(s.SequenceId, out var pos))
{
// overwrite with new pending state
StateEntity existing = states[pos].Value;
existing.TransactionId = s.TransactionId;
existing.TransactionTimestamp = s.TimeStamp;
existing.TransactionManager = JsonConvert.SerializeObject(s.TransactionManager, this.jsonSettings);
existing.SetState(s.State, this.jsonSettings);
await batchOperation.Add(new TableTransactionAction(TableTransactionActionType.UpdateReplace, existing.Entity, existing.ETag)).ConfigureAwait(false);
key.ETag = batchOperation.KeyETag;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace($"{partition}.{existing.RowKey} Update {existing.TransactionId}");
}
}
else
{
var entity = StateEntity.Create(this.jsonSettings, this.partition, s);
await batchOperation.Add(new TableTransactionAction(TableTransactionActionType.Add, entity.Entity)).ConfigureAwait(false);
key.ETag = batchOperation.KeyETag;
states.Insert(pos, new KeyValuePair <long, StateEntity>(s.SequenceId, entity));
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace($"{partition}.{entity.RowKey} Insert {entity.TransactionId}");
}
}
}
}
}
// third, persist metadata and commit position
key.Metadata = JsonConvert.SerializeObject(metadata, this.jsonSettings);
if (commitUpTo.HasValue && commitUpTo.Value > key.CommittedSequenceId)
{
key.CommittedSequenceId = commitUpTo.Value;
}
if (string.IsNullOrEmpty(this.key.ETag.ToString()))
{
await batchOperation.Add(new TableTransactionAction(TableTransactionActionType.Add, key)).ConfigureAwait(false);
key.ETag = batchOperation.KeyETag;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace($"{partition}.{KeyEntity.RK} Insert. v{this.key.CommittedSequenceId}, {metadata.CommitRecords.Count}c");
}
}
else
{
await batchOperation.Add(new TableTransactionAction(TableTransactionActionType.UpdateReplace, key, key.ETag)).ConfigureAwait(false);
key.ETag = batchOperation.KeyETag;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace($"{partition}.{KeyEntity.RK} Update. v{this.key.CommittedSequenceId}, {metadata.CommitRecords.Count}c");
}
}
// fourth, remove obsolete records
if (states.Count > 0 && states[0].Key < obsoleteBefore)
{
FindState(obsoleteBefore, out var pos);
for (int i = 0; i < pos; i++)
{
await batchOperation.Add(new TableTransactionAction(TableTransactionActionType.Delete, states[i].Value.Entity, states[i].Value.ETag)).ConfigureAwait(false);
key.ETag = batchOperation.KeyETag;
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace($"{partition}.{states[i].Value.RowKey} Delete {states[i].Value.TransactionId}");
}
}
states.RemoveRange(0, pos);
}
await batchOperation.Flush().ConfigureAwait(false);
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug($"{partition} Stored v{this.key.CommittedSequenceId} eTag={key.ETag}");
}
return(key.ETag.ToString());
}
public async Task <string> Store(string expectedETag, TransactionalStateMetaData metadata, List <PendingTransactionState <TState> > statesToPrepare, long?commitUpTo, long?abortAfter)
{
if (this.StateStorage.Etag != expectedETag)
{
throw new ArgumentException(nameof(expectedETag), "Etag does not match");
}
stateStorage.State.Metadata = metadata;
var pendinglist = stateStorage.State.PendingStates;
// abort
if (abortAfter.HasValue && pendinglist.Count != 0)
{
var pos = pendinglist.FindIndex(t => t.SequenceId > abortAfter.Value);
if (pos != -1)
{
pendinglist.RemoveRange(pos, pendinglist.Count - pos);
}
}
// prepare
if (statesToPrepare?.Count > 0)
{
foreach (var p in statesToPrepare)
{
var pos = pendinglist.FindIndex(t => t.SequenceId >= p.SequenceId);
if (pos == -1)
{
pendinglist.Add(p); //append
}
else if (pendinglist[pos].SequenceId == p.SequenceId)
{
pendinglist[pos] = p; //replace
}
else
{
pendinglist.Insert(pos, p); //insert
}
}
}
// commit
if (commitUpTo.HasValue && commitUpTo.Value > stateStorage.State.CommittedSequenceId)
{
var pos = pendinglist.FindIndex(t => t.SequenceId == commitUpTo.Value);
if (pos != -1)
{
var committedState = pendinglist[pos];
stateStorage.State.CommittedSequenceId = committedState.SequenceId;
stateStorage.State.CommittedState = committedState.State;
pendinglist.RemoveRange(0, pos + 1);
}
else
{
throw new InvalidOperationException($"Transactional state corrupted. Missing prepare record (SequenceId={commitUpTo.Value}) for committed transaction.");
}
}
await stateStorage.WriteStateAsync();
return(stateStorage.Etag);
}
public async Task <string> Store(string expectedETag, TransactionalStateMetaData metadata,
List <PendingTransactionState <TState> > statesToPrepare, long?commitUpTo,
long?abortAfter)
{
if (_metadata.ETag != expectedETag)
{
throw new ArgumentException(nameof(expectedETag), "Etag does not match");
}
var stateUpdater = new StateUpdater(_dbExecuter, _stateId, _options.StateTableName, _jsonSettings);
// first, clean up aborted records
if (abortAfter.HasValue && _states.Count != 0)
{
await stateUpdater.DeleteStates(afterSequenceId : abortAfter);
while (_states.Count > 0 && _states[_states.Count - 1].SequenceId > abortAfter)
{
_states.RemoveAt(_states.Count - 1);
}
}
// second, persist non-obsolete prepare records
var obsoleteBefore = commitUpTo.HasValue ? commitUpTo.Value : _metadata.CommittedSequenceId;
if (statesToPrepare != null && statesToPrepare.Count > 0)
{
foreach (var s in statesToPrepare)
{
if (s.SequenceId >= obsoleteBefore)
{
var newState = new TransactionStateEntity
{
SequenceId = s.SequenceId,
TransactionId = s.TransactionId,
Timestamp = s.TimeStamp,
TransactionManager = s.TransactionManager,
Value = s.State
};
if (FindState(s.SequenceId, out var pos))
{
// overwrite with new pending state
_states[pos] = newState;
await stateUpdater.UpdateState(newState).ConfigureAwait(false);
}
else
{
await stateUpdater.InsertState(newState).ConfigureAwait(false);
_states.Insert(pos, newState);
}
_logger.LogTrace($"{_stateId}.{newState.SequenceId} Update {newState.TransactionId}");
}
}
}
await stateUpdater.EnsureInsertBufferFlushed();
// third, persist metadata and commit position
_metadata = await PersistMetadata(metadata, commitUpTo ?? _metadata.CommittedSequenceId);
// fourth, remove obsolete records
if (_states.Count > 0 && _states[0].SequenceId < obsoleteBefore)
{
FindState(obsoleteBefore, out var pos);
await stateUpdater.DeleteStates(beforeSequenceId : obsoleteBefore);
_states.RemoveRange(0, pos);
}
return(_metadata.ETag);
}
protected abstract Task <ITransactionMetadataEntity> PersistMetadata(TransactionalStateMetaData value,
long commitSequenceId);
public Task <string> Store(string expectedETag, TransactionalStateMetaData metadata, List <PendingTransactionState <TState> > statesToPrepare, long?commitUpTo, long?abortAfter)
{
throw new NotImplementedException();
}
