private long GetOldestNonAckedMessageTimestamp(PeerState peer)
{
var peerId = peer.PeerId.ToString();
var lastAckedMessageTimestamp = 0L;
foreach (var currentBucketId in BucketIdHelper.GetBucketsCollection(peer.OldestNonAckedMessageTimestampInTicks))
{
var messagesInBucket = _dataContext.PersistentMessages
.Where(x => x.PeerId == peerId &&
x.BucketId == currentBucketId &&
x.UniqueTimestampInTicks >= peer.OldestNonAckedMessageTimestampInTicks)
.OrderBy(x => x.UniqueTimestampInTicks)
.Select(x => new { x.IsAcked, x.UniqueTimestampInTicks })
.Execute();
foreach (var message in messagesInBucket)
{
if (!message.IsAcked)
{
return(message.UniqueTimestampInTicks);
}
lastAckedMessageTimestamp = message.UniqueTimestampInTicks;
}
}
return(lastAckedMessageTimestamp == 0 ? SystemDateTime.UtcNow.Ticks : lastAckedMessageTimestamp + 1);
}
public void should_not_iterate_over_future_buckets()
{
var now = DateTime.UtcNow;
var beginning = now.AddHours(3);
BucketIdHelper.GetBucketsCollection(beginning.Ticks).ShouldBeEmpty();
}
public void should_return_correct_bucket_id()
{
var now = DateTime.UtcNow;
var expectedBucketId = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0).Ticks;
BucketIdHelper.GetBucketId(now).ShouldEqual(expectedBucketId);
}
private Task RemovePersistentMessages(PeerId peerId)
{
var allPossibleBuckets = BucketIdHelper.GetBucketsCollection(SystemDateTime.UtcNow.Ticks - CqlStorage.PersistentMessagesTimeToLive.Ticks).ToArray();
return(_dataContext.PersistentMessages
.Where(x => x.PeerId == peerId.ToString() && allPossibleBuckets.Contains(x.BucketId))
.Delete()
.ExecuteAsync());
}
private void InsertPersistentMessage(PeerId peerId, DateTime timestamp, AckState ackState = AckState.Acked)
{
var message = new PersistentMessage
{
PeerId = peerId.ToString(),
BucketId = BucketIdHelper.GetBucketId(timestamp),
IsAcked = ackState == AckState.Acked,
UniqueTimestampInTicks = timestamp.Ticks,
TransportMessage = new byte[0]
};
DataContext.PersistentMessages.Insert(message).Execute();
}
private void InsertPersistentMessage(PeerId peerId, DateTime timestamp, Action <PersistentMessage> updateMessage = null)
{
var message = new PersistentMessage
{
PeerId = peerId.ToString(),
BucketId = BucketIdHelper.GetBucketId(timestamp),
IsAcked = true,
UniqueTimestampInTicks = timestamp.Ticks,
TransportMessage = new byte[0]
};
updateMessage?.Invoke(message);
DataContext.PersistentMessages.Insert(message).Execute();
}
public void should_iterate_over_buckets_since_a_given_timestamp_and_stop_when_hitting_current_bucket()
{
var now = DateTime.UtcNow;
var beginning = now.AddHours(-3);
var expectedBucketIds = new[]
{
BucketIdHelper.GetBucketId(beginning),
BucketIdHelper.GetBucketId(beginning.AddHours(1)),
BucketIdHelper.GetBucketId(beginning.AddHours(2)),
BucketIdHelper.GetBucketId(now),
};
BucketIdHelper.GetBucketsCollection(beginning.Ticks).ShouldBeEquivalentTo(expectedBucketIds);
}
private void CleanBuckets(PeerId peerId, long previousOldestMessageTimestamp, long newOldestMessageTimestamp)
{
var firstBucketToDelete = BucketIdHelper.GetBucketId(previousOldestMessageTimestamp);
var lastBucketToDelete = BucketIdHelper.GetPreviousBucketId(newOldestMessageTimestamp);
if (firstBucketToDelete == lastBucketToDelete)
{
return;
}
var bucketsToDelete = BucketIdHelper.GetBucketsCollection(firstBucketToDelete, lastBucketToDelete).ToArray();
_dataContext.PersistentMessages
.Where(x => x.PeerId == peerId.ToString() && bucketsToDelete.Contains(x.BucketId))
.Delete()
.ExecuteAsync();
}
public async Task should_store_messages_in_different_buckets()
{
MessageId.ResetLastTimestamp();
var firstTime = DateTime.Now;
using (SystemDateTime.Set(firstTime))
using (MessageId.PauseIdGenerationAtDate(firstTime))
{
var peerId = new PeerId("Abc.Testing.Target");
var firstMessageId = MessageId.NextId();
await _storage.Write(new[] { MatcherEntry.Message(peerId, firstMessageId, new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });
var secondTime = firstTime.AddHours(1);
SystemDateTime.Set(secondTime);
MessageId.PauseIdGenerationAtDate(secondTime);
var secondMessageId = MessageId.NextId();
await _storage.Write(new[] { MatcherEntry.Message(peerId, secondMessageId, new MessageTypeId("Abc.OtherMessage"), new byte[] { 0x04, 0x05, 0x06 }) });
var persistedMessages = DataContext.PersistentMessages.Execute().OrderBy(x => x.UniqueTimestampInTicks).ToList(); // Results are only ordered withing a partition
persistedMessages.Count.ShouldEqual(2);
persistedMessages.First().ShouldHaveSamePropertiesAs(new PersistentMessage
{
BucketId = BucketIdHelper.GetBucketId(firstTime),
PeerId = peerId.ToString(),
IsAcked = false,
MessageId = firstMessageId.Value,
TransportMessage = new byte[] { 0x01, 0x02, 0x03 },
UniqueTimestampInTicks = firstTime.Ticks
});
persistedMessages.Last().ShouldHaveSamePropertiesAs(new PersistentMessage
{
BucketId = BucketIdHelper.GetBucketId(secondTime),
PeerId = peerId.ToString(),
IsAcked = false,
MessageId = secondMessageId.Value,
TransportMessage = new byte[] { 0x04, 0x05, 0x06 },
UniqueTimestampInTicks = secondTime.Ticks
});
}
}
public async Task should_delete_all_buckets_for_peer_when_removed()
{
var now = SystemDateTime.UtcNow;
_peerStateRepository.UpdateNonAckMessageCount(new PeerId("PeerId"), 0);
DataContext.PersistentMessages.Insert(new PersistentMessage
{
BucketId = BucketIdHelper.GetBucketId(now),
IsAcked = true,
PeerId = "PeerId",
TransportMessage = new byte[0],
UniqueTimestampInTicks = now.Ticks
}).Execute();
DataContext.PersistentMessages.Execute().Count().ShouldEqual(1);
await _peerStateRepository.RemovePeer(new PeerId("PeerId"));
DataContext.PersistentMessages.Execute().Any().ShouldBeFalse();
}
public IEnumerable <TransportMessage> GetUnackedMessages()
{
var oldestNonAckedMessageTimestampInTicks = _peerState.OldestNonAckedMessageTimestampInTicks;
_log.Info($"Reading messages for peer {_peerState.PeerId} from {oldestNonAckedMessageTimestampInTicks} ({new DateTime(oldestNonAckedMessageTimestampInTicks).ToLongTimeString()})");
var nonAckedMessagesInBuckets = BucketIdHelper.GetBucketsCollection(oldestNonAckedMessageTimestampInTicks)
.Select(b => GetNonAckedMessagesInBucket(oldestNonAckedMessageTimestampInTicks, b));
var nonAckedMessageRead = 0;
foreach (var nonAckedMessagesInBucket in nonAckedMessagesInBuckets)
{
foreach (var nonAckedMessage in nonAckedMessagesInBucket)
{
nonAckedMessageRead++;
yield return(nonAckedMessage);
}
}
_log.Info($"{nonAckedMessageRead} non acked messages replayed for peer {_peerState.PeerId}");
}
public Task Write(IList <MatcherEntry> entriesToPersist)
{
if (entriesToPersist.Count == 0)
{
return(Task.CompletedTask);
}
var fattestMessage = entriesToPersist.OrderByDescending(msg => msg.MessageBytes?.Length ?? 0).First();
_reporter.AddStorageReport(entriesToPersist.Count, entriesToPersist.Sum(msg => msg.MessageBytes?.Length ?? 0), fattestMessage.MessageBytes?.Length ?? 0, fattestMessage.MessageTypeName);
var countByPeer = new Dictionary <PeerId, int>();
foreach (var matcherEntry in entriesToPersist)
{
var shouldInvestigatePeer = _configuration.PeerIdsToInvestigate != null && _configuration.PeerIdsToInvestigate.Contains(matcherEntry.PeerId.ToString());
if (shouldInvestigatePeer)
{
_log.Info($"Storage requested for peer {matcherEntry.PeerId}, Type: {matcherEntry.Type}, Message Id: {matcherEntry.MessageId}");
}
var countDelta = matcherEntry.IsAck ? -1 : 1;
countByPeer[matcherEntry.PeerId] = countDelta + countByPeer.GetValueOrDefault(matcherEntry.PeerId);
if (shouldInvestigatePeer)
{
_log.Info($"Count delta computed for peer {matcherEntry.PeerId}, will increment: {countDelta}");
}
}
var insertTasks = new List <Task>();
var remaining = new SemaphoreSlim(_maxParallelInsertTasks);
foreach (var matcherEntry in entriesToPersist)
{
var gotSlot = remaining.Wait(TimeSpan.FromSeconds(10));
if (!gotSlot)
{
_log.Warn("Could not get slot to insert in cassandra after 10 second.");
}
var messageDateTime = matcherEntry.MessageId.GetDateTimeForV2OrV3();
var rowTimestamp = matcherEntry.IsAck ? messageDateTime.AddTicks(10) : messageDateTime;
var boundStatement = _preparedStatement.Bind(matcherEntry.PeerId.ToString(),
BucketIdHelper.GetBucketId(messageDateTime),
messageDateTime.Ticks,
matcherEntry.MessageId.Value,
matcherEntry.IsAck,
matcherEntry.MessageBytes,
(int)PeerState.MessagesTimeToLive.TotalSeconds,
ToUnixMicroSeconds(rowTimestamp));
var insertTask = _dataContext.Session.ExecuteAsync(boundStatement);
insertTasks.Add(insertTask);
insertTask.ContinueWith(t =>
{
var shouldInvestigatePeer = _configuration.PeerIdsToInvestigate != null && _configuration.PeerIdsToInvestigate.Contains(matcherEntry.PeerId.ToString());
if (shouldInvestigatePeer)
{
_log.Info($"Storage done for peer {matcherEntry.PeerId}, Type: {matcherEntry.Type}, Message Id: {matcherEntry.MessageId}, TaskResult: {t.Status}");
}
if (t.IsFaulted)
{
_log.Error(t.Exception);
}
remaining.Release();
}, TaskContinuationOptions.ExecuteSynchronously);
}
var updateNonAckedCountTasks = new List <Task>();
foreach (var countForPeer in countByPeer)
{
updateNonAckedCountTasks.Add(_peerStateRepository.UpdateNonAckMessageCount(countForPeer.Key, countForPeer.Value));
}
return(Task.WhenAll(insertTasks.Concat(updateNonAckedCountTasks)));
}
public Task Write(IList <MatcherEntry> entriesToPersist)
{
if (entriesToPersist.Count == 0)
{
return(Task.CompletedTask);
}
var fattestMessage = entriesToPersist.OrderByDescending(msg => msg.MessageBytes?.Length ?? 0).First();
_reporter.AddStorageReport(entriesToPersist.Count, entriesToPersist.Sum(msg => msg.MessageBytes?.Length ?? 0), fattestMessage.MessageBytes?.Length ?? 0, fattestMessage.MessageTypeName);
var insertTasks = new List <Task>(entriesToPersist.Count);
var countByPeer = new Dictionary <PeerId, int>();
foreach (var matcherEntry in entriesToPersist)
{
var shouldInvestigatePeer = _configuration.PeerIdsToInvestigate != null && _configuration.PeerIdsToInvestigate.Contains(matcherEntry.PeerId.ToString());
if (shouldInvestigatePeer)
{
_log.Info($"Storage requested for peer {matcherEntry.PeerId}, Type: {matcherEntry.Type}, Message Id: {matcherEntry.MessageId}");
}
var messageDateTime = matcherEntry.MessageId.GetDateTimeForV2OrV3();
var rowTimestamp = matcherEntry.IsAck ? messageDateTime.AddTicks(10) : messageDateTime;
var insertTask = _parallelPersistor.Insert(_preparedStatement.Bind(matcherEntry.PeerId.ToString(),
BucketIdHelper.GetBucketId(messageDateTime),
messageDateTime.Ticks,
matcherEntry.MessageId.Value,
matcherEntry.IsAck,
matcherEntry.MessageBytes,
(int)PersistentMessagesTimeToLive.TotalSeconds,
ToUnixMicroSeconds(rowTimestamp)));
if (shouldInvestigatePeer)
{
insertTask = insertTask.ContinueWith(t => _log.Info($"Storage done for peer {matcherEntry.PeerId}, Type: {matcherEntry.Type}, Message Id: {matcherEntry.MessageId}, TaskResult: {t.Status}"));
}
insertTasks.Add(insertTask);
var countDelta = matcherEntry.IsAck ? -1 : 1;
countByPeer[matcherEntry.PeerId] = countDelta + countByPeer.GetValueOrDefault(matcherEntry.PeerId);
if (shouldInvestigatePeer)
{
_log.Info($"Count delta computed for peer {matcherEntry.PeerId}, will increment: {countDelta}");
}
}
foreach (var countForPeer in countByPeer)
{
_peerStateRepository.UpdateNonAckMessageCount(countForPeer.Key, countForPeer.Value);
}
return(Task.WhenAll(insertTasks));
}