public async Task AbleToMigrate_fails_if_Timeout_table_Does_not_exist()
{
// Arrange
var timeoutTarget = new ASQTarget(connectionString, new DelayedDeliveryTableNameProvider("TimeoutTableThatDoesNotExist"));
// Act
var ableToMigrate = await timeoutTarget.AbleToMigrate(new EndpointInfo { EndpointName = endpointName });
// Assert
Assert.IsFalse(ableToMigrate.CanMigrate);
Assert.AreEqual("Target delayed delivery table TimeoutTableThatDoesNotExist does not exist.", ableToMigrate.Problems[0]);
}
public async Task Prepare_creates_the_staging_queue()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
var timeoutTarget = new ASQTarget(connectionString, nameProvider);
// Act
await using var migrator = await timeoutTarget.PrepareTargetEndpointBatchMigrator(endpointName);
// Assert
Assert.IsTrue(await DoesTableExist(nameProvider.GetStagingTableName(endpointName)).ConfigureAwait(true));
}
public async Task AbleToMigrate_passes_if_Timeout_table_exists()
{
// Arrange
await CreateTimeoutTable($"T{endpointName}");
var timeoutTarget = new ASQTarget(connectionString, new DelayedDeliveryTableNameProvider($"T{endpointName}"));
// Act
var ableToMigrate = await timeoutTarget.AbleToMigrate(new EndpointInfo { EndpointName = endpointName });
// Assert
Assert.IsTrue(ableToMigrate.CanMigrate);
}
public async Task Completing_with_large_entities_batches_respecting_size_limitations()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
var endpointName = nameof(Completing_with_large_entities_batches_respecting_size_limitations);
var cutOffDate = DateTime.UtcNow;
var random = new Random();
var timeouts = new List <StagedDelayedMessageEntity>();
var target = new ASQTarget(connectionString, nameProvider);
var stagingTableName = nameProvider.GetStagingTableName(endpointName);
await CreateTimeoutTable(stagingTableName);
await CreateTimeoutTable(nameProvider.GetDelayedDeliveryTableName(endpointName));
var stagingTable = tableClient.GetTableReference(stagingTableName);
// the entity will roughly be 98 KB and we will store 50 of those which makes the actual payload be around 5 MB
for (var x = 0; x < 50; x++)
{
var dateTime = cutOffDate.AddDays(random.Next(1, 5));
var messageId = Guid.NewGuid().ToString();
var entity = new StagedDelayedMessageEntity
{
Destination = new string('a', 32 * 1024),
Headers = new string('a', 32 * 1024),
Time = dateTime,
MessageId = messageId,
Body = new byte[32 * 1024],
RowKey = $"{messageId}_{dateTime.ToString($"yyyyMMddHHmmss")}",
PartitionKey = "1"
};
await stagingTable.ExecuteAsync(TableOperation.Insert(entity));
timeouts.Add(entity);
}
// Act
var result = await target.PrepareTargetEndpointBatchMigrator(endpointName);
var completeResult = await result.CompleteBatch(1);
// Assert
Assert.IsNotNull(completeResult);
Assert.AreEqual(timeouts.Count, completeResult);
}
public async Task Complete_Removes_Staging_Queue_If_Empty()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
await CreateTimeoutTable(nameProvider.GetDelayedDeliveryTableName(endpointName));
var timeoutTarget = new ASQTarget(connectionString, nameProvider);
await using var migrator = await timeoutTarget.PrepareTargetEndpointBatchMigrator(endpointName);
await migrator.StageBatch(new List <TimeoutData>
{
new TimeoutData
{
Id = "SomeID",
Headers = new Dictionary <string, string>
{
{ "NServiceBus.MessageId", "SomeMessageId" }
},
Destination = "SomeDestination",
State = new byte[2],
Time = new DateTime(2021, 12, 12, 12, 12, 12, DateTimeKind.Utc)
},
new TimeoutData
{
Id = "SomeOtherId",
Headers = new Dictionary <string, string>
{
{ "NServiceBus.MessageId", "SomeOtherMessageId" }
},
Destination = "SomeOtherDestination",
State = new byte[2],
Time = new DateTime(2021, 12, 12, 12, 13, 13, DateTimeKind.Utc)
},
}, 1);
await migrator.CompleteBatch(1);
// Act
await timeoutTarget.Complete(endpointName);
// Assert
var stagingTableExists = await DoesTableExist(nameProvider.GetStagingTableName(endpointName));
Assert.IsFalse(stagingTableExists);
}
public async Task CompleteBatch_Moves_All_Entries_From_Staging_To_DelayedMessageTable()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
await CreateTimeoutTable(nameProvider.GetDelayedDeliveryTableName(endpointName));
var timeoutTarget = new ASQTarget(connectionString, nameProvider);
await using var migrator = await timeoutTarget.PrepareTargetEndpointBatchMigrator(endpointName);
await migrator.StageBatch(new List <TimeoutData>
{
new TimeoutData
{
Id = "SomeID",
Headers = new Dictionary <string, string>
{
{ "NServiceBus.MessageId", "SomeMessageId" }
},
Destination = "SomeDestination",
State = new byte[2],
Time = new DateTime(2021, 12, 12, 12, 12, 12, DateTimeKind.Utc)
},
new TimeoutData
{
Id = "SomeOtherId",
Headers = new Dictionary <string, string>
{
{ "NServiceBus.MessageId", "SomeOtherMessageId" }
},
Destination = "SomeOtherDestination",
State = new byte[2],
Time = new DateTime(2021, 12, 12, 12, 13, 13, DateTimeKind.Utc)
},
}, 1);
// Act
var numberCompleted = await migrator.CompleteBatch(1);
// Assert
var recordsInTimeoutTable = await ReadTimeoutsFromTable(nameProvider.GetDelayedDeliveryTableName(endpointName));
Assert.AreEqual(2, recordsInTimeoutTable.Count);
Assert.AreEqual(2, numberCompleted);
}
public async Task AbleToMigrate_fails_with_incorrect_connection_string()
{
// Arrange
string fakeConnectionString = "DefaultEndpointsProtocol=https;AccountName=fakename;AccountKey=g94OvNO9o3sVan5eipemQEHmU8zD2M9iq98E8nKSdR2bTuQB1hi07Yd1/8dDw6+1jGI2klWjpvoDahHPhR/3og==";
var account = CloudStorageAccount.Parse(fakeConnectionString);
var client = account.CreateCloudTableClient();
client.DefaultRequestOptions.MaximumExecutionTime = TimeSpan.FromSeconds(2);
var timeoutTarget = new ASQTarget(client, new DelayedDeliveryTableNameProvider("TimeoutTableThatDoesNotExist"));
// Act
var ableToMigrate = await timeoutTarget.AbleToMigrate(new EndpointInfo { EndpointName = endpointName });
// Assert
Assert.IsFalse(ableToMigrate.CanMigrate);
Assert.That(ableToMigrate.Problems[0], Does.StartWith("Unable to connect to the storage instance on account 'fakename'. Verify the connection string. Exception message '"));
}
public async Task Complete_Throws_If_Messages_Are_Still_Staged()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
await CreateTimeoutTable(nameProvider.GetDelayedDeliveryTableName(endpointName));
var timeoutTarget = new ASQTarget(connectionString, nameProvider);
await using var migrator = await timeoutTarget.PrepareTargetEndpointBatchMigrator(endpointName);
var numberStaged = await migrator.StageBatch(new List <TimeoutData>
{
new TimeoutData
{
Id = "SomeID",
Headers = new Dictionary <string, string>
{
{ "NServiceBus.MessageId", "SomeMessageId" }
},
Destination = "SomeDestination",
State = new byte[2],
Time = new DateTime(2021, 12, 12, 12, 12, 12, DateTimeKind.Utc)
},
new TimeoutData
{
Id = "SomeOtherId",
Headers = new Dictionary <string, string>
{
{ "NServiceBus.MessageId", "SomeOtherMessageId" }
},
Destination = "SomeOtherDestination",
State = new byte[2],
Time = new DateTime(2021, 12, 12, 12, 13, 13, DateTimeKind.Utc)
},
}, 1);
// Assert
Assert.ThrowsAsync <Exception>(async() =>
{
// Act
await timeoutTarget.Complete(endpointName);
});
}
public async Task Abort_Removes_Staging_Table()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
await CreateTimeoutTable(nameProvider.GetDelayedDeliveryTableName(endpointName));
var timeoutTarget = new ASQTarget(connectionString, nameProvider);
await using var migrator = await timeoutTarget.PrepareTargetEndpointBatchMigrator(endpointName);
// Act
await timeoutTarget.Abort(endpointName);
// Assert
var stagingTableExists = await DoesTableExist(nameProvider.GetStagingTableName(endpointName));
Assert.IsFalse(stagingTableExists);
}
public async Task Staging_with_large_entities_batches_respecting_size_limitations()
{
// Arrange
var nameProvider = new DelayedDeliveryTableNameProvider();
var endpointName = nameof(Staging_with_large_entities_batches_respecting_size_limitations);
var cutOffDate = DateTime.UtcNow;
var random = new Random();
var target = new ASQTarget(connectionString, nameProvider);
var timeouts = new List <TimeoutData>();
// the entity will roughly be 98 KB and we will store 50 of those which makes the actual payload be around 5 MB
for (var x = 0; x < 50; x++)
{
var dateTime = cutOffDate.AddDays(random.Next(1, 5));
timeouts.Add(new TimeoutData
{
Destination = new string('a', 32 * 1024),
Headers = new Dictionary <string, string> {
{ "a", new string('h', 16 * 1024) }
},
Time = dateTime,
Id = Guid.NewGuid().ToString(),
State = new byte[32 * 1024],
SagaId = Guid.NewGuid(),
OwningTimeoutManager = this.endpointName
});
}
// Act
var result = await target.PrepareTargetEndpointBatchMigrator(endpointName);
var stageResult = await result.StageBatch(timeouts, 1);
// Assert
Assert.IsNotNull(stageResult);
Assert.AreEqual(timeouts.Count, stageResult);
}
public async Task Can_migrate_timeouts()
{
var sourceEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(RavenDBSource));
var targetEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(AsqTarget));
var ravenTimeoutPrefix = "TimeoutDatas";
var ravenVersion = RavenDbVersion.Four;
var ravenAdapter = new Raven4Adapter(serverUrl, databaseName);
await Scenario.Define <SourceContext>()
.WithEndpoint <RavenDBSource>(b => b.CustomConfig(ec =>
{
ec.UsePersistence <RavenDBPersistence>()
.SetDefaultDocumentStore(GetDocumentStore(serverUrl, databaseName));
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(session, c) =>
{
var delayedMessage = new DelayedMessage();
var options = new SendOptions();
options.DelayDeliveryWith(TimeSpan.FromSeconds(20));
options.SetDestination(targetEndpoint);
await session.Send(delayedMessage, options);
await WaitUntilTheTimeoutIsSavedInRaven(ravenAdapter, sourceEndpoint);
c.TimeoutSet = true;
}))
.Done(c => c.TimeoutSet)
.Run(TimeSpan.FromSeconds(15));
var context = await Scenario.Define <TargetContext>()
// Create the legacy endpoint to forward the delayed message to the native delayed delivery endpoint
// This is needed as ASQ stores the delayed messages at the sending endpoint until delivery is needed
.WithEndpoint <RavenDBSource>(b => b.CustomConfig(ec =>
{
var transport = ec.UseTransport <AzureStorageQueueTransport>().ConnectionString(asqConnectionString);
transport.DisablePublishing();
transport.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
}))
.WithEndpoint <AsqTarget>(b => b.CustomConfig(ec =>
{
var transport = ec.UseTransport <AzureStorageQueueTransport>().ConnectionString(asqConnectionString);
transport.DisablePublishing();
transport.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(_, c) =>
{
var logger = new TestLoggingAdapter(c);
var timeoutsSource = new RavenDbTimeoutsSource(logger, serverUrl, databaseName, ravenTimeoutPrefix, ravenVersion, false);
var timeoutsTarget = new ASQTarget(asqConnectionString, new DelayedDeliveryTableNameProvider());
var migrationRunner = new MigrationRunner(logger, timeoutsSource, timeoutsTarget);
await migrationRunner.Run(DateTime.Now.AddDays(-1), EndpointFilter.SpecificEndpoint(sourceEndpoint), new Dictionary <string, string>());
}))
.Done(c => c.GotTheDelayedMessage)
.Run(TimeSpan.FromSeconds(30));
Assert.True(context.GotTheDelayedMessage);
}
public async Task Can_migrate_timeouts()
{
var sourceEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(AspSource));
var targetEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(AsqTarget));
await Scenario.Define <SourceContext>()
.WithEndpoint <AspSource>(b => b.CustomConfig(ec =>
{
SetupPersistence(ec);
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(session, c) =>
{
var delayedMessage = new DelayedMessage();
var options = new SendOptions();
options.DelayDeliveryWith(TimeSpan.FromSeconds(15));
options.SetDestination(targetEndpoint);
await session.Send(delayedMessage, options);
await WaitUntilTheTimeoutsAreSavedInAsp(sourceEndpoint, 2);
c.TimeoutSet = true;
}))
.Done(c => c.TimeoutSet)
.Run(TimeSpan.FromSeconds(30));
var context = await Scenario.Define <TargetContext>()
// Create the legacy endpoint to forward the delayed message to the reporting endpoint
// This is needed as ASQ stores the delayed messages at the sending endpoint until
// delivery is needed
.WithEndpoint <AspSource>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
}))
// Start the reporting endpoint to receive and process the delayed message
.WithEndpoint <AsqTarget>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(_, c) =>
{
var logger = new TestLoggingAdapter(c);
var timeoutStorage = CreateTimeoutStorage(sourceEndpoint);
var timeoutsTarget = new ASQTarget(asqConnectionString, new DelayedDeliveryTableNameProvider());
var migrationRunner = new MigrationRunner(logger, timeoutStorage, timeoutsTarget);
await migrationRunner.Run(DateTime.Now.AddDays(-10), EndpointFilter.SpecificEndpoint(sourceEndpoint), new Dictionary <string, string>());
}))
.Done(c => c.GotTheDelayedMessage)
.Run(TimeSpan.FromSeconds(30));
Assert.True(context.GotTheDelayedMessage);
}
public async Task Can_migrate_timeouts()
{
var salesEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(SqlPSource));
var reportingEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(AsqTarget));
// Make sure delayed delivery queue is created so that the migration can run.
await Scenario.Define <SourceContext>()
.WithEndpoint <SqlPSource>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
})).Run(TimeSpan.FromSeconds(10));
// Sending a delayed delivery message using TimeoutManager
await Scenario.Define <SourceContext>()
.WithEndpoint <SqlPSource>(b => b.CustomConfig(ec =>
{
var persistence = ec.UsePersistence <SqlPersistence>();
persistence.SubscriptionSettings().DisableCache();
persistence.SqlDialect <NServiceBus.SqlDialect.MsSqlServer>();
persistence.ConnectionBuilder(
connectionBuilder: () => new SqlConnection(connectionString));
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(session, c) =>
{
var delayedMessage = new DelayedMessage();
var options = new SendOptions();
options.DelayDeliveryWith(TimeSpan.FromSeconds(15));
options.SetDestination(reportingEndpoint);
await session.Send(delayedMessage, options);
await WaitUntilTheTimeoutIsSavedInSql(salesEndpoint);
c.TimeoutSet = true;
}))
.Done(c => c.TimeoutSet)
.Run(TimeSpan.FromSeconds(30));
var context = await Scenario.Define <TargetContext>()
// Create the sales endpoint to forward the delayed message to the reporting endpoint
// This is needed as ASQ stores the delayed messages at the sending endpoint until
// delivery is needed
.WithEndpoint <SqlPSource>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
}))
// Start the reporting endpoint to receieve and process the delayed message
.WithEndpoint <AsqTarget>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(_, c) =>
{
var logger = new TestLoggingAdapter(c);
var timeoutStorage = new SqlTimeoutsSource(connectionString, new MsSqlServer(), 1024);
var timeoutsTarget = new ASQTarget(asqConnectionString, new DelayedDeliveryTableNameProvider());
var migrationRunner = new MigrationRunner(logger, timeoutStorage, timeoutsTarget);
await migrationRunner.Run(DateTime.Now.AddDays(-10), EndpointFilter.SpecificEndpoint(salesEndpoint), new Dictionary <string, string>());
}))
.Done(c => c.GotTheDelayedMessage)
.Run(TimeSpan.FromSeconds(30));
Assert.True(context.GotTheDelayedMessage);
}
public async Task Can_migrate_timeouts()
{
var sourceEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(NHibernateSource));
var targetEndpoint = NServiceBus.AcceptanceTesting.Customization.Conventions.EndpointNamingConvention(typeof(AsqTarget));
using (var testSession = CreateSessionFactory().OpenSession())
{ // Explicit using scope to ensure dispose before SUT connects
using (var testTx = testSession.BeginTransaction())
{
await testSession.SaveAsync(new TimeoutEntity
{
Endpoint = sourceEndpoint,
Destination = targetEndpoint,
SagaId = Guid.NewGuid(),
Headers = "{\"NServiceBus.EnclosedMessageTypes\": \"TimeoutMigrationTool.NHibernate.AcceptanceTests.NHibernateToAsqEndToEnd+DelayedMessage\"}",
State = Encoding.UTF8.GetBytes("{}"),
Time = DateTime.UtcNow.AddSeconds(15)
});
await testTx.CommitAsync();
}
}
var context = await Scenario.Define <TargetContext>()
// Create the legacy endpoint to forward the delayed message to the reporting endpoint
// This is needed as ASQ stores the delayed messages at the sending endpoint until
// delivery is needed
.WithEndpoint <NHibernateSource>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
}))
.WithEndpoint <AsqTarget>(b => b.CustomConfig(ec =>
{
var transportConfig = ec.UseTransport <AzureStorageQueueTransport>();
transportConfig.ConnectionString(asqConnectionString);
transportConfig.DisablePublishing();
transportConfig.DelayedDelivery().DisableTimeoutManager();
ec.UseSerialization <NewtonsoftSerializer>();
})
.When(async(_, c) =>
{
var logger = new TestLoggingAdapter(c);
var timeoutsSource = new NHibernateTimeoutsSource(connectionString, 512, DatabaseDialect);
var timeoutsTarget = new ASQTarget(asqConnectionString, new DelayedDeliveryTableNameProvider());
var migrationRunner = new MigrationRunner(logger, timeoutsSource, timeoutsTarget);
await migrationRunner.Run(DateTime.Now.AddDays(-10), EndpointFilter.SpecificEndpoint(sourceEndpoint), new Dictionary <string, string>());
}))
.Done(c => c.GotTheDelayedMessage)
.Run(TimeSpan.FromSeconds(90));
Assert.True(context.GotTheDelayedMessage);
}
