public void Migrate(MigrateOptions options)
{
Prepare(options, "Migrate");
PrintMigrationGroup(options);
var crunner = new Runner(SharpFactory.Default.CreateDataClient(), GetAssemblyWithMigrations(options), options.MigrationGroup);
crunner.Run(options.TargetVersion);
}
public async Task Run(MigrateOptions options)
{
try
{
Guard.ArgumentNotNull(options, nameof(options));
await SetThroughPut(100000);
await RollBackCosmosDocuments(options);
}
finally
{
await SetThroughPut(2000);
}
/*
* Due to incorrect configuration being set for the variation pointers in prod, the DSG specification has profiling incorrectly published.
* To correct the profiles in this version, we need to roll back to the previously published state as of March. To further complicate the data, a refresh has been performed which has incorrectly flattened the profiles in an updated state.
* Create another console app in the backend solution to perform the migration, like CalculateFunding.Migrations.PublishedProviderPopulateReleased.
* PublishedFunding
* Rollback each of the DSG PublishedFunding documents in cosmos to the previous major version.
* Replace c.content.current with the contents of the previous version based on c.content.current.majorVersion - 1. Lookup the PublishedFundingVersion based on the previous major version.
* PublishedProvider
* Rollback each of the DSG PublishedProvider documents in cosmos to the second latest released version, eg
* PublishedProvider.content.released.majorVersion - 1
* (NOTE: see all versions with this query: where c.documentType='PublishedProviderVersion' and c.content.fundingStreamId = 'DSG' and c.content.status = 'Released' and c.content.providerId = '10006547')
* eg for provider 10006547 the major version should be 3.0 as the current released version is publishedprovider-10006547-FY-2021-DSG-12
* Replace c.contents.current and c.content.released in the PublishedProvider document with the contents of the PublishedProviderVersion above.
* Testing and development
* Copy the publishedfunding cosmos collection from prod into a new collection in the dev cosmos to perform development against using the dt.exe tool.
* Once the documents have been reverted, we will need to run this against preprod to test with the current version of DSG and CFS software. It should override the blob contents and search indexes when republished for the already existing versions. A cache clear in the external API may be required.
*
* NOTE: the models for publishedproviders/published funding in the providers branch will be ahead of what's deployed into preprod/prod at the moment but should be backwards compatible.
* 1. Order of deploying to an environment:
* 2. Run the migration tool to revert DSG back to March
* 3. Reindex published providers search
* 4. Ensure provider totals are correct as per March
* 5. Set variation pointer correctly
* 6. Refresh funding
* 7. Check profiling totals
* 8. Approve funding
* 9. Release funding
* 10. Reindex published providers
* 11. Clear external API cache key
* 12. Ensure output for DSG is correct in external API
*
*
*/
}
public Task KeyMigrateAsync(RedisKey key, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None)
{
return(Inner.KeyMigrateAsync(ToInner(key), toServer, toDatabase, timeoutMilliseconds, migrateOptions, flags));
}
public void KeyMigrate(RedisKey key, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None)
{
Inner.KeyMigrate(ToInner(key), toServer, toDatabase, timeoutMilliseconds, migrateOptions, flags);
}
public DsgRollBackParameters(MigrateOptions migrateOptions)
{
FundingPeriodId = migrateOptions.FundingPeriod;
Version = migrateOptions.DocumentVersion;
}
static int RunMigrate(ILogger logger, MigrateOptions options) => Migrate.Run(logger);
public Task KeyMigrateAsync(RedisKey key, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0,
MigrateOptions migrateOptions = MigrateOptions.None,
CommandFlags flags = CommandFlags.None) =>
this._database.KeyMigrateAsync(key, toServer, toDatabase, timeoutMilliseconds, migrateOptions, flags);
public void Migrate(string keySuffix, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None)
{
InitRedisKey(keySuffix);
var command = RedisGroup.GetCommand(true, Key);
command.KeyMigrate(Key, toServer, toDatabase, timeoutMilliseconds, migrateOptions, (sr.CommandFlags)flags);
}
public async Task Run(MigrateOptions migrateOptions)
{
DsgRollBackParameters rollBackParameters = migrateOptions;
await PerformUndo(rollBackParameters);
}
void IDatabase.KeyMigrate(RedisKey key, EndPoint toServer, int toDatabase, int timeoutMilliseconds, MigrateOptions migrateOptions, CommandFlags flags) => Multiplexer.Wait(((IDatabaseAsync)this).KeyMigrateAsync(key, toServer, toDatabase, timeoutMilliseconds, migrateOptions, flags));
static int RunMigrate(MigrateOptions opts) => Migrate.Run();
public void Migrate(string keySuffix, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None)
{
throw new NotImplementedException();
}
/// <summary>
/// MIGRATE : https://redis.io/commands/migrate
/// </summary>
public static Task Migrate <T>(this T redis, EndPoint toServer, int toDatabase = 0, int timeoutMilliseconds = 0, MigrateOptions migrateOptions = MigrateOptions.None, CommandFlags flags = CommandFlags.None)
where T : IRedisStructure
=> redis.Connection.Database.KeyMigrateAsync(redis.Key, toServer, toDatabase, timeoutMilliseconds, migrateOptions, flags);
public MigrateCommand(MigrateOptions options, BoundConfig config)
{
_options = options;
_config = config;
}
private static void MigrateMain(MigrateOptions opt)
{
try
{
var(logon, chars, world) = Program.ConfigLoader.Load <MigrationConfig>("migrate").MergeAll();
// TODO: only logon db migration implemented right now
if (logon != null)
{
var mapper = new LogonDatabaseMapper();
Migrate(logon, mapper);
}
}
catch (ArgumentException ex)
{
Console.Error.WriteLine(ex.Message);
return;
}
catch (MySqlException ex)
{
Console.Error.WriteLine(ex.Message);
return;
}
void Migrate <TSource, TDestination>(MigrationSettings settings, DatabaseMapper <TSource, TDestination> mapper)
where TDestination : Entities.LiteDb.Database
where TSource : DbContext
{
if (File.Exists(settings.LiteDatabase))
{
Console.Write("Destination {0} already exists. Overwrite it? [y/N] ", settings.LiteDatabase);
var answer = Regex.IsMatch(
Console.ReadLine(),
"^(y(es?)?)$",
RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture
);
if (!answer)
{
Console.WriteLine("Skipping {0} ({1}) migration", settings.MySqlDatabase, settings.LiteDatabase);
return;
}
}
Console.WriteLine("Migrating {0} database to {1}", settings.MySqlDatabase, settings.LiteDatabase);
using (var memory = new MemoryStream())
using (var service = new StreamDiskService(memory))
using (var litedb = (TDestination)Activator.CreateInstance(typeof(TDestination), service, settings.LitePassword, null as BsonMapper))
using (var mysql = (TSource)Activator.CreateInstance(typeof(TSource), settings))
{
Mapper.Initialize(mapper.ConfigurMapping);
mapper.MigrateDatabase(mysql, litedb);
Console.WriteLine(" - Flushing {0} to disk...", settings.LiteDatabase);
service.Flush();
memory.Flush();
var file = File.Open(settings.LiteDatabase, FileMode.Create, FileAccess.Write, FileShare.None);
using ( file )
{
memory.Position = 0;
memory.CopyTo(file);
file.Flush();
litedb.Flush();
}
}
}
}
private async Task RollBackCosmosDocuments(MigrateOptions migrateOptions)
{
await _undoCosmosJob.Run(migrateOptions);
}