static void RecordKnownEndpoints(EndpointDetails observedEndpoint, Dictionary <string, KnownEndpoint> observedEndpoints, ProcessedMessage processedMessage)
{
var uniqueEndpointId = $"{observedEndpoint.Name}{observedEndpoint.HostId}";
if (!observedEndpoints.TryGetValue(uniqueEndpointId, out var knownEndpoint))
{
knownEndpoint = new KnownEndpoint
{
Host = observedEndpoint.Host,
HostId = observedEndpoint.HostId,
LastSeen = processedMessage.ProcessedAt,
Name = observedEndpoint.Name,
Id = KnownEndpoint.MakeDocumentId(observedEndpoint.Name, observedEndpoint.HostId),
};
observedEndpoints.Add(uniqueEndpointId, knownEndpoint);
}
knownEndpoint.LastSeen = processedMessage.ProcessedAt > knownEndpoint.LastSeen ? processedMessage.ProcessedAt : knownEndpoint.LastSeen;
}
public async Task Should_noop_migration_if_no_index()
{
Guid sendHostId = Guid.NewGuid();
Guid receiverHostId = Guid.NewGuid();
using (var store = InMemoryStoreBuilder.GetInMemoryStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new ProcessedMessage
{
Id = "1",
ProcessedAt = DateTime.Now,
UniqueMessageId = "xyz",
MessageMetadata = new System.Collections.Generic.Dictionary <string, object>
{
{ "SendingEndpoint", new EndpointDetails {
Host = "SendHost", HostId = sendHostId, Name = "SendHostName"
} },
{ "ReceivingEndpoint", new EndpointDetails {
Host = "ReceivingHost", HostId = receiverHostId, Name = "ReceivingHostName"
} },
}
});
await session.SaveChangesAsync();
}
store.WaitForIndexing();
var migrator = new MigrateKnownEndpoints(store);
await migrator.MigrateEndpoints(pageSize : 1).ConfigureAwait(false);
using (var session = store.OpenAsyncSession())
{
var loadedSenderEndpoint = await session.LoadAsync <KnownEndpoint>(KnownEndpoint.MakeDocumentId("SendHostName", sendHostId)).ConfigureAwait(false);
Assert.Null(loadedSenderEndpoint);
}
}
}
public void AddEndpoint(KnownEndpoint endpoint, IBucketer bucketer)
{
Endpoints.Add(endpoint);
Bucketers.Add(endpoint.Request.OriginalEndpoint, bucketer);
}
internal async Task MigrateEndpoints(int pageSize = 1024)
{
var knownEndpointsIndex = await store.AsyncDatabaseCommands.GetIndexAsync("EndpointsIndex").ConfigureAwait(false);
if (knownEndpointsIndex == null)
{
Logger.Debug("EndpointsIndex migration already completed.");
// Index has already been deleted, no need to migrate
return;
}
var dbStatistics = await store.AsyncDatabaseCommands.GetStatisticsAsync().ConfigureAwait(false);
var indexStats = dbStatistics.Indexes.First(index => index.Name == knownEndpointsIndex.Name);
if (indexStats.Priority == IndexingPriority.Disabled)
{
Logger.Debug("EndpointsIndex already disabled. Deleting EndpointsIndex.");
// This should only happen the second time the migration is attempted.
// The index is disabled so the data should have been migrated. We can now delete the index.
await store.AsyncDatabaseCommands.DeleteIndexAsync(knownEndpointsIndex.Name).ConfigureAwait(false);
return;
}
int previouslyDone = 0;
do
{
using (var session = store.OpenAsyncSession())
{
var endpointsFromIndex = await session.Query <dynamic>(knownEndpointsIndex.Name, true)
.Skip(previouslyDone)
.Take(pageSize)
.ToListAsync()
.ConfigureAwait(false);
if (endpointsFromIndex.Count == 0)
{
Logger.Debug("No more records from EndpointsIndex to migrate.");
break;
}
previouslyDone += endpointsFromIndex.Count;
var knownEndpoints = endpointsFromIndex.Select(endpoint => new KnownEndpoint
{
Id = KnownEndpoint.MakeDocumentId(endpoint.Name, Guid.Parse(endpoint.HostId)),
Host = endpoint.Host,
HostId = Guid.Parse(endpoint.HostId),
Name = endpoint.Name,
LastSeen = DateTime.UtcNow // Set the imported date to be now since we have no better guess
});
using (var bulkInsert = store.BulkInsert(options: new BulkInsertOptions
{
OverwriteExisting = true
}))
{
foreach (var endpoint in knownEndpoints)
{
bulkInsert.Store(endpoint);
}
Logger.Debug($"Migrating {endpointsFromIndex.Count} entries.");
await bulkInsert.DisposeAsync().ConfigureAwait(false);
}
}
} while (true);
Logger.Debug("EndpointsIndex entries migrated. Disabling EndpointsIndex.");
// Disable the index so it can be safely deleted in the next migration run
await store.AsyncDatabaseCommands.SetIndexPriorityAsync(knownEndpointsIndex.Name, IndexingPriority.Disabled).ConfigureAwait(false);
}
public void Handle(RegisterEndpoint message)
{
var id = message.EndpointInstanceId;
//Injecting store in this class because we want to know about ConcurrencyExceptions so that EndpointsCache.MarkAsProcessed is not called if the save fails.
using (var session = Store.OpenSession())
{
session.Advanced.UseOptimisticConcurrency = true;
KnownEndpoint knownEndpoint;
if (id == Guid.Empty)
{
knownEndpoint = session.Query <KnownEndpoint, KnownEndpointIndex>().SingleOrDefault(e => e.EndpointDetails.Name == message.Endpoint.Name && e.EndpointDetails.Host == message.Endpoint.Host);
}
else
{
knownEndpoint = session.Load <KnownEndpoint>(id);
if (knownEndpoint == null)
{
knownEndpoint = session.Query <KnownEndpoint, KnownEndpointIndex>().SingleOrDefault(e => e.HasTemporaryId &&
e.EndpointDetails.Name == message.Endpoint.Name &&
e.EndpointDetails.Host == message.Endpoint.Host);
}
}
if (knownEndpoint == null)
{
//new endpoint
Bus.Publish(new NewEndpointDetected
{
Endpoint = message.Endpoint,
DetectedAt = message.DetectedAt
});
knownEndpoint = new KnownEndpoint
{
EndpointDetails = message.Endpoint,
HostDisplayName = message.Endpoint.Host,
Monitored = message.EnableMonitoring
};
if (id == Guid.Empty)
{
knownEndpoint.Id = DeterministicGuid.MakeId(message.Endpoint.Name, message.Endpoint.HostId.ToString());
knownEndpoint.HasTemporaryId = true;
}
else
{
knownEndpoint.Id = id;
}
session.Store(knownEndpoint);
}
else
{
if (knownEndpoint.HasTemporaryId && id != Guid.Empty)
{
session.Delete(knownEndpoint);
session.Store(new KnownEndpoint
{
Id = id,
EndpointDetails = message.Endpoint,
HostDisplayName = message.Endpoint.Host,
Monitored = knownEndpoint.Monitored || message.EnableMonitoring
});
}
knownEndpoint.Monitored |= message.EnableMonitoring;
}
session.SaveChanges();
}
}
