private async Task <int> ArchieveUnreferencedBinaries(
IEnumerable <KafkaEvent <SessionCreatingEvent> > expiredSessionEvents,
ICollection <Guid> sessionsWithReferences,
CancellationToken cancellationToken)
{
var count = 0;
foreach (var expiredSessionEvent in expiredSessionEvents)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var sessionId = expiredSessionEvent.Source.SessionId;
if (sessionsWithReferences.Contains(sessionId))
{
continue;
}
var sessionExpiresAt = expiredSessionEvent.Source.ExpiresAt;
var fileCount = await _sessionCleanupService.ArchiveSessionAsync(sessionId, sessionExpiresAt);
if (fileCount > 0)
{
_logger.LogInformation(
"Session '{sessionId}' cleaned up as expired and unused. File count: '{fileCount}'. " +
"Session expiration date: {sessionExpiresAt:u}",
sessionId,
fileCount,
sessionExpiresAt);
}
else
{
_logger.LogWarning("Session '{sessionId}' not found.", sessionId);
}
await _sessionsEventReceiver.CommitAsync(expiredSessionEvent);
++count;
}
return(count);
}
private IDisposable BinaryReferenceEventsProducing(CancellationToken cancellationToken)
{
async Task ProcessAsync(KafkaEvent <ObjectVersionCreatingEvent> @event)
{
var objectId = @event.Source.ObjectId;
var versionId = @event.Source.CurrentVersionId;
IReadOnlyCollection <ObjectVersionRecord> versionRecords;
if (string.IsNullOrEmpty(versionId))
{
var policy = CreateGetObjectVersionsResiliencePolicy();
versionRecords = await policy.ExecuteAsync(() => _objectsStorageReader.GetObjectVersions(objectId, versionId));
if (versionRecords == null)
{
_logger.LogWarning(
"{taskName}: Got an event for the object with id = '{objectId}' that was not eventually created. The event will be skipped.",
nameof(ObjectVersionCreatedEventsProducing),
objectId);
return;
}
}
else
{
versionRecords = await _objectsStorageReader.GetObjectVersions(objectId, versionId);
}
_logger.LogInformation(
"{taskName}: There are '{versionsCount}' new versions were created after the versionId = {versionId} " +
"for the object id = '{objectId}'.",
nameof(BinaryReferenceEventsProducing),
versionRecords.Count,
versionId,
objectId);
foreach (var record in versionRecords)
{
var fileInfos = record.Elements
.Where(x => x.Value is IBinaryElementValue binaryValue && !string.IsNullOrEmpty(binaryValue.Raw))
.Select(x => (TemplateCode: x.TemplateCode, FileKey: ((IBinaryElementValue)x.Value).Raw))
.ToList();
foreach (var fileInfo in fileInfos)
{
await _eventSender.SendAsync(
_binariesUsingsTopic,
new BinaryReferencedEvent(objectId, record.VersionId, fileInfo.TemplateCode, fileInfo.FileKey, record.LastModified));
_logger.LogInformation(
"{taskName}: Event for binary reference {fileKey} for element with templateCode = '{templateCode}' " +
"for object id = '{objectId}' and versionId = {versionId} sent to {topic}.",
nameof(BinaryReferenceEventsProducing),
fileInfo.FileKey,
fileInfo.TemplateCode,
record.Id,
record.VersionId,
_binariesUsingsTopic);
}
}
await _binariesEventReceiver.CommitAsync(@event);
}
var observable = _binariesEventReceiver.Subscribe <ObjectVersionCreatingEvent>(cancellationToken);
return(observable
.Do(@event =>
{
var retry = CreateRetryPolicy(_logger, nameof(BinaryReferenceEventsProducing));
retry.Execute(() => ProcessAsync(@event).GetAwaiter().GetResult());
})
.Subscribe());
}
private IDisposable ObjectVersionCreatedEventsProducing(CancellationToken cancellationToken)
{
async Task ProcessAsync(KafkaEvent <ObjectVersionCreatingEvent> @event)
{
var objectId = @event.Source.ObjectId;
var versionId = @event.Source.CurrentVersionId;
IReadOnlyCollection <ObjectVersionRecord> versionRecords;
if (string.IsNullOrEmpty(versionId))
{
var policy = CreateGetObjectVersionsResiliencePolicy();
versionRecords = await policy.ExecuteAsync(() => _objectsStorageReader.GetObjectVersions(objectId, versionId));
if (versionRecords == null)
{
_logger.LogWarning(
"{taskName}: Got an event for the object with id = '{objectId}' that was not eventually created. The event will be skipped.",
nameof(ObjectVersionCreatedEventsProducing),
objectId);
return;
}
}
else
{
versionRecords = await _objectsStorageReader.GetObjectVersions(objectId, versionId);
}
_logger.LogInformation(
"{taskName}: There are '{versionsCount}' new versions were created after the versionId = {versionId} for the object id = '{objectId}'.",
nameof(ObjectVersionCreatedEventsProducing),
versionRecords.Count,
versionId,
objectId);
foreach (var record in versionRecords)
{
var versionCreatedEvent = new ObjectVersionCreatedEvent(
record.Id,
record.VersionId,
record.VersionIndex,
record.Author,
record.Properties,
record.LastModified);
await _eventSender.SendAsync(_objectVersionsTopic, versionCreatedEvent);
_logger.LogInformation(
"{taskName}: Event for object id = '{objectId}' and versionId = {versionId} sent to {topic}.",
nameof(ObjectVersionCreatedEventsProducing),
record.Id,
record.VersionId,
_objectVersionsTopic);
}
await _versionEventReceiver.CommitAsync(@event);
}
var observable = _versionEventReceiver.Subscribe <ObjectVersionCreatingEvent>(cancellationToken);
return(observable
.Do(@event =>
{
var retry = CreateRetryPolicy(_logger, nameof(ObjectVersionCreatedEventsProducing));
retry.Execute(() => ProcessAsync(@event).GetAwaiter().GetResult());
})
.Subscribe());
}