private static TableQuery <EventTagEntry> GenerateTaggedMessageQuery(
ReplayTaggedMessages replay)
{
var partitionKeyFilter =
TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.Equal,
PartitionKeyEscapeHelper.Escape(EventTagEntry.GetPartitionKey(replay.Tag)));
var utcTicksTRowKeyFilter =
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.GreaterThan,
$"{replay.FromOffset.ToJournalRowKey()}{EventTagEntry.AsciiIncrementedDelimiter}"),
TableOperators.And,
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.LessThanOrEqual,
$"{replay.ToOffset.ToJournalRowKey()}{EventTagEntry.Delimiter}"));
var filter =
TableQuery.CombineFilters(
partitionKeyFilter,
TableOperators.And,
utcTicksTRowKeyFilter);
var returnValue = new TableQuery <EventTagEntry>().Where(filter);
return(returnValue);
}
public void Should_escape_correctly(string partitionKey)
{
var escapedKey = PartitionKeyEscapeHelper.Escape(partitionKey);
escapedKey.Should().NotContain("/");
var originalKey = PartitionKeyEscapeHelper.Unescape(escapedKey);
originalKey.Should().Be(partitionKey);
}
private static TableQuery <PersistentJournalEntry> GeneratePersistentJournalEntryReplayQuery(
string persistentId,
long fromSequenceNumber,
long toSequenceNumber)
{
var persistenceIdFilter =
TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.Equal,
PartitionKeyEscapeHelper.Escape(persistentId));
var highestSequenceNrFilter =
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.NotEqual,
HighestSequenceNrEntry.RowKeyValue);
var filter =
TableQuery.CombineFilters(
persistenceIdFilter,
TableOperators.And,
highestSequenceNrFilter);
if (fromSequenceNumber > 0)
{
filter =
TableQuery.CombineFilters(
filter,
TableOperators.And,
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.GreaterThanOrEqual,
fromSequenceNumber.ToJournalRowKey()));
}
if (toSequenceNumber != long.MaxValue)
{
filter =
TableQuery.CombineFilters(
filter,
TableOperators.And,
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.LessThanOrEqual,
toSequenceNumber.ToJournalRowKey()));
}
var returnValue = new TableQuery <PersistentJournalEntry>().Where(filter);
return(returnValue);
}
private static TableQuery <HighestSequenceNrEntry> GenerateHighestSequenceNumberQuery(
string persistenceId)
{
var filter =
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.Equal,
PartitionKeyEscapeHelper.Escape(persistenceId)),
TableOperators.And,
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.Equal,
HighestSequenceNrEntry.RowKeyValue));
var returnValue = new TableQuery <HighestSequenceNrEntry>().Where(filter);
return(returnValue);
}
//private static TableQuery GeneratePersistentJournalEntryDeleteQuery(
private static TableQuery <PersistentJournalEntry> GeneratePersistentJournalEntryDeleteQuery(
string persistenceId,
long toSequenceNr)
{
var persistenceIdFilter =
TableQuery.GenerateFilterCondition(
"PartitionKey",
QueryComparisons.Equal,
PartitionKeyEscapeHelper.Escape(persistenceId));
var highestSequenceNrFilter =
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.NotEqual,
HighestSequenceNrEntry.RowKeyValue);
var rowKeyLessThanFilter =
TableQuery.GenerateFilterCondition(
"RowKey",
QueryComparisons.LessThanOrEqual,
toSequenceNr.ToJournalRowKey());
var rowKeyFilter =
TableQuery.CombineFilters(
highestSequenceNrFilter,
TableOperators.And,
rowKeyLessThanFilter);
var filter =
TableQuery.CombineFilters(
persistenceIdFilter,
TableOperators.And,
rowKeyFilter);
var returnValue = new TableQuery <PersistentJournalEntry>().Where(filter);
return(returnValue);
}
protected override async Task <IImmutableList <Exception> > WriteMessagesAsync(
IEnumerable <AtomicWrite> atomicWrites)
{
try
{
var taggedEntries = ImmutableDictionary <string, List <EventTagEntry> > .Empty;
var exceptions = ImmutableList <Exception> .Empty;
var highSequenceNumbers = ImmutableDictionary <string, long> .Empty;
using (var currentWrites = atomicWrites.GetEnumerator())
{
while (currentWrites.MoveNext())
{
Debug.Assert(currentWrites.Current != null, "atomicWrites.Current != null");
var list = currentWrites.Current.Payload.AsInstanceOf <IImmutableList <IPersistentRepresentation> >();
var batchItems = ImmutableList <ITableEntity> .Empty;
foreach (var t in list)
{
var item = t;
Debug.Assert(item != null, nameof(item) + " != null");
byte[] payloadBytes = null;
string[] tags = {};
// If the payload is a tagged payload, reset to a non-tagged payload
if (item.Payload is Tagged tagged)
{
item = item.WithPayload(tagged.Payload);
payloadBytes = _serialization.PersistentToBytes(item);
if (tagged.Tags.Count > 0)
{
tags = tagged.Tags.ToArray();
}
}
if (payloadBytes == null)
{
payloadBytes = _serialization.PersistentToBytes(item);
}
var newItem =
new PersistentJournalEntry(
item.PersistenceId,
item.SequenceNr,
payloadBytes,
item.Manifest,
tags);
batchItems = batchItems.Add(newItem);
foreach (var tag in tags)
{
if (!taggedEntries.ContainsKey(tag))
{
taggedEntries = taggedEntries.SetItem(tag, new List <EventTagEntry>());
}
taggedEntries[tag].Add(
new EventTagEntry(
newItem.PartitionKey,
tag,
newItem.SeqNo,
newItem.Payload,
newItem.Manifest,
newItem.UtcTicks));
}
highSequenceNumbers =
highSequenceNumbers.SetItem(
item.PersistenceId,
item.SequenceNr);
}
try
{
var persistenceBatch = new TableBatchOperation();
highSequenceNumbers.ForEach(
x => batchItems = batchItems.Add(
new HighestSequenceNrEntry(x.Key, x.Value)));
// Encode partition keys for writing
foreach (var tableEntity in batchItems)
{
tableEntity.PartitionKey = PartitionKeyEscapeHelper.Escape(tableEntity.PartitionKey);
}
batchItems.ForEach(x => persistenceBatch.InsertOrReplace(x));
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
_log.Debug("Attempting to write batch of {0} messages to Azure storage", persistenceBatch.Count);
}
var persistenceResults = await Table.ExecuteBatchAsLimitedBatches(persistenceBatch);
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
foreach (var r in persistenceResults)
{
_log.Debug("Azure table storage wrote entity [{0}] with status code [{1}]", r.Etag, r.HttpStatusCode);
}
}
exceptions = exceptions.Add(null);
}
catch (Exception ex)
{
_log.Warning(ex, "Failure while writing messages to Azure table storage");
exceptions = exceptions.Add(ex);
}
}
}
if (exceptions.All(ex => ex == null))
{
var allPersistenceIdsBatch = new TableBatchOperation();
highSequenceNumbers.ForEach(x =>
{
var encodedKey = PartitionKeyEscapeHelper.Escape(x.Key);
allPersistenceIdsBatch.InsertOrReplace(new AllPersistenceIdsEntry(encodedKey));
});
var allPersistenceResults = await Table.ExecuteBatchAsLimitedBatches(allPersistenceIdsBatch);
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
foreach (var r in allPersistenceResults)
{
_log.Debug("Azure table storage wrote entity [{0}] with status code [{1}]", r.Etag, r.HttpStatusCode);
}
}
if (HasPersistenceIdSubscribers || HasAllPersistenceIdSubscribers)
{
highSequenceNumbers.ForEach(x => NotifyNewPersistenceIdAdded(x.Key));
}
if (taggedEntries.Count > 0)
{
var eventTagsBatch = new TableBatchOperation();
foreach (var kvp in taggedEntries)
{
eventTagsBatch.Clear();
foreach (var item in kvp.Value)
{
item.PartitionKey = PartitionKeyEscapeHelper.Escape(item.PartitionKey);
eventTagsBatch.InsertOrReplace(item);
}
var eventTagsResults = await Table.ExecuteBatchAsLimitedBatches(eventTagsBatch);
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
foreach (var r in eventTagsResults)
{
_log.Debug("Azure table storage wrote entity [{0}] with status code [{1}]", r.Etag, r.HttpStatusCode);
}
}
if (HasTagSubscribers && taggedEntries.Count != 0)
{
foreach (var tag in taggedEntries.Keys)
{
NotifyTagChange(tag);
}
}
}
}
}
/*
* Part of the Akka.Persistence design.
*
* Either return null or return an exception for each failed AtomicWrite.
*
* Either everything fails or everything succeeds is the idea I guess.
*/
return(exceptions.Any(ex => ex != null) ? exceptions : null);
}
catch (Exception ex)
{
_log.Error(ex, "Error during WriteMessagesAsync");
throw;
}
}
public override async Task ReplayMessagesAsync(
IActorContext context,
string persistenceId,
long fromSequenceNr,
long toSequenceNr,
long max,
Action <IPersistentRepresentation> recoveryCallback)
{
NotifyNewPersistenceIdAdded(persistenceId);
_log.Debug("Entering method ReplayMessagesAsync for persistentId [{0}] from seqNo range [{1}, {2}] and taking up to max [{3}]", persistenceId, fromSequenceNr, toSequenceNr, max);
if (max == 0)
{
return;
}
var replayQuery = GeneratePersistentJournalEntryReplayQuery(persistenceId, fromSequenceNr, toSequenceNr);
var nextTask = Table.ExecuteQuerySegmentedAsync(replayQuery, null);
var count = 0L;
while (nextTask != null)
{
var tableQueryResult = await nextTask;
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
_log.Debug("Recovered [{0}] messages for entity [{1}]", tableQueryResult.Results.Count, persistenceId);
}
if (tableQueryResult.ContinuationToken != null)
{
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
_log.Debug("Have additional messages to download for entity [{0}]", persistenceId);
}
// start the next query while we process the results of this one
nextTask = Table.ExecuteQuerySegmentedAsync(replayQuery, tableQueryResult.ContinuationToken);
}
else
{
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
_log.Debug("Completed download of messages for entity [{0}]", persistenceId);
}
// terminates the loop
nextTask = null;
}
foreach (var savedEvent in tableQueryResult.Results)
{
// check if we've hit max recovery
if (count >= max)
{
return;
}
++count;
var deserialized = _serialization.PersistentFromBytes(savedEvent.Payload);
// Write the new persistent because it sets the sender as deadLetters which is not correct
var persistent =
new Persistent(
deserialized.Payload,
deserialized.SequenceNr,
PartitionKeyEscapeHelper.Unescape(deserialized.PersistenceId),
deserialized.Manifest,
deserialized.IsDeleted,
ActorRefs.NoSender,
deserialized.WriterGuid,
timestamp: savedEvent.UtcTicks);
if (_log.IsDebugEnabled && _settings.VerboseLogging)
{
_log.Debug("Recovering [{0}] for entity [{1}].", persistent, savedEvent.PartitionKey);
}
recoveryCallback(persistent);
}
}
_log.Debug("Leaving method ReplayMessagesAsync");
}
