public ChangeFeedProcessorHost(MigrationConfig config)
{
this.config = config ?? throw new ArgumentNullException(nameof(config));
this.SourcePartitionKeys = config.SourcePartitionKeys;
this.TargetPartitionKey = config.TargetPartitionKey;
this.leaseCollectionClient = KeyVaultHelper.Singleton.CreateCosmosClientFromKeyVault(
EnvironmentConfig.Singleton.MigrationMetadataCosmosAccountName,
Program.MigrationClientUserAgentPrefix,
useBulk: false,
retryOn429Forever: true);
this.sourceCollectionClient = KeyVaultHelper.Singleton.CreateCosmosClientFromKeyVault(
config.MonitoredAccount,
Program.SourceClientUserAgentPrefix,
useBulk: false,
retryOn429Forever: true);
this.destinationCollectionClient = KeyVaultHelper.Singleton.CreateCosmosClientFromKeyVault(
config.DestAccount,
Program.DestinationClientUserAgentPrefix,
useBulk: true,
retryOn429Forever: true);
this.deadletterClient = KeyVaultHelper.Singleton.GetBlobContainerClientFromKeyVault(
EnvironmentConfig.Singleton.DeadLetterAccountName,
config.Id?.ToLowerInvariant().Replace("-", String.Empty));
// Make sure to allow multiple active migrations for the same source container
// by creating a unique processor name for every config document
this.processorName = config.ProcessorName;
}
public async Task <IActionResult> OnPostSubmitAsync(
string id,
string sourceAccount,
string sourceDatabase,
string sourceContainer,
string sourcePK,
string destinationAccount,
string destinationDatabase,
string destinationContainer,
string destinationPK)
{
MigrationConfig newConfig = new MigrationConfig
{
Id = id,
MonitoredAccount = sourceAccount,
MonitoredDbName = sourceDatabase,
MonitoredCollectionName = sourceContainer,
SourcePartitionKeys = sourcePK,
DestAccount = destinationAccount,
DestDbName = destinationDatabase,
DestCollectionName = destinationContainer,
TargetPartitionKey = destinationPK,
Completed = false,
StartTimeEpochMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
};
await MigrationConfigDal.Singleton
.CreateMigrationAsync(newConfig)
.ConfigureAwait(false);
return(this.RedirectToPage("/Migrations", null));
}
public async Task batch_get_next_for_migration_all_events()
{
var entity = Guid.NewGuid();
await _store.Append(Setup.UnversionedCommit(guid: entity));
await _store.Append(Setup.UnversionedCommit(guid: entity));
await _store.Append(Setup.UnversionedCommit("1"));
var rm = new MigrationConfig("test");
var rez = _store.GetNextBatch(rm, 0);
rez.IsEmpty.Should().BeFalse();
var first = rez.GetNext().Value;
first.EntityId.Should().Be(entity);
first.TenantId.Should().Be("_");
first.Version.Should().Be(1);
var second = rez.GetNext().Value;
second.EntityId.Should().Be(entity);
second.TenantId.Should().Be("_");
second.Version.Should().Be(2);
var third = rez.GetNext().Value;
third.EntityId.Should().NotBe(entity);
third.TenantId.Should().Be("1");
third.Version.Should().Be(1);
rez.GetNext().IsEmpty.Should().BeTrue();
}
public MigrationModel(IOptions <MigrationConfig> config,
INotificationImportRepository notificationImportRepository)
{
_config = config.Value;
_notificationImportRepository = notificationImportRepository;
ValidationService = new ValidationService(this);
}
public async Task <MigrationConfig> CreateMigrationAsync(MigrationConfig config)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (String.IsNullOrWhiteSpace(config.Id))
{
config.Id = Guid.NewGuid().ToString("N");
}
try
{
BlobContainerClient deadletterClient = KeyVaultHelper.Singleton.GetBlobContainerClientFromKeyVault(
EnvironmentConfig.Singleton.DeadLetterAccountName,
config.Id?.ToLowerInvariant().Replace("-", String.Empty));
await deadletterClient
.CreateIfNotExistsAsync(PublicAccessType.None)
.ConfigureAwait(false);
return(await this.container
.CreateItemAsync <MigrationConfig>(config, new PartitionKey(config.Id))
.ConfigureAwait(false));
}
catch (Exception error)
{
TelemetryHelper.Singleton.LogError(
"MigrationConfigDal.CreateMigrationAsync for document with id {0} failed: {1}",
config.Id,
error);
throw;
}
}
/// <inheritdoc />
public virtual void MigrationMetadataChanged(IMigrationMetadata migrationMetadata, IExtendedMigrationProcessor extendedMigrationProcessor)
{
Processor = extendedMigrationProcessor;
if (!Enabled)
{
return;
}
if (!Processor.SchemaExists(CommonSchemaName) ||
!Processor.SchemaExists(AuthCodeSchemaName))
{
return;
}
var logonSql = MigrationConfig
.PrepareSql(SqlResources.LogonScriptSql)
.ReplaceIgnoreCase("{CommonSchemaName}", CommonSchemaName)
.ReplaceIgnoreCase("{AuthCodeSchemaName}", AuthCodeSchemaName);
Execute(logonSql);
if (Processor.SchemaExists(MigrationConfig.Schema))
{
ProcessAfter(new CreateSchemaWithPrefixExpression
{
SchemaName = MigrationConfig.Schema,
SchemaPrefixId = MigrationConfig.GetSchemaPrefixId(),
SchemaPrefixUniqueId = MigrationConfig.GetSchemaPrefixUniqueId()
});
}
}
public CommittedEvents GetNextBatch(MigrationConfig config, ProcessedCommitsCount count)
{
IEnumerable <Commit> all = _commits;
if (!config.TenantId.IsNullOrEmpty())
{
all = all.Where(d => d.TenantId == config.TenantId);
}
all = all.OrderBy(d => d.Timestamp);
return(new CommittedEvents(all.ToArray()));
}
public void ReleasesCalculate([FromBody] MigrationConfig migrationConfig)
{
Dictionary <string, decimal> currencies = db.Currencies.ToDictionary(currency => currency.CurrencyCode, currency => currency.ConversionFactorUYU);
var query = db.Releases.Include(release => release.ReleaseItems);
foreach (var release in query)
{
release.TotalAmountUYU = (int)CalculateTotal(release, currencies);
}
db.SaveChanges();
}
public override void ExecuteCmdlet()
{
try
{
if (string.IsNullOrEmpty(LegacyConfigId))
{
var migrationPlanList = StorSimpleClient.GetAllMigrationPlan();
if (migrationPlanList.MigrationPlans.Count == 0)
{
WriteWarning(Resources.MigrationPlanNoConfigs);
}
else
{
foreach (var migrationPlan in migrationPlanList.MigrationPlans)
{
var migrationPlanConfig = new MigrationConfig(migrationPlan);
WriteObject(migrationPlanConfig);
}
}
}
else
{
StorSimpleClient.UpdateMigrationPlanSync(LegacyConfigId);
var migrationPlanList = StorSimpleClient.GetMigrationPlan(LegacyConfigId);
if (0 >= migrationPlanList.MigrationPlans.Count)
{
throw new ArgumentException(Resources.MigrationPlanNotFound);
}
else
{
MigrationPlan migrationPlan = migrationPlanList.MigrationPlans.First();
if (null != LegacyContainerNames)
{
var legacyContainerNamesList = LegacyContainerNames.ToList();
migrationPlan.MigrationPlanInfo =
migrationPlan.MigrationPlanInfo.ToList().FindAll(
plan =>
legacyContainerNamesList.Contains(plan.DataContainerName,
StringComparer.InvariantCultureIgnoreCase));
}
var migrationPlanMsg = new MigrationPlanMsg(migrationPlan);
WriteObject(migrationPlanMsg);
}
}
}
catch (Exception except)
{
this.HandleException(except);
}
}
public CommittedEvents GetNextBatch(MigrationConfig config, ProcessedCommitsCount count)
{
using (var db = _db.Create())
{
var all = db.QueryAs(q => q
.FromAnonymous(new { Id = 1, TenantId = "", EntityId = Guid.Empty },
new TableName(CommitsTable, Schema)).Where(d => true)
.AndIf(() => !config.TenantId.IsNullOrEmpty(), d => d.TenantId == config.TenantId)
.OrderBy(d => d.Id)
.Limit(config.BatchSize, count.Value)
.SelectAll(useAsterisk: true).MapTo <Commit>()
);
return(new CommittedEvents(all.ToArray()));
}
}
public async Task <MigrationConfig> CompleteMigrationAsync(string id)
{
try
{
while (true)
{
MigrationConfig config = await this.GetMigrationAsync(id).ConfigureAwait(false);
if (config.Completed)
{
return(config);
}
config.Completed = true;
try
{
return(await this.container
.ReplaceItemAsync <MigrationConfig>(
config,
config.Id,
new PartitionKey(config.Id),
new ItemRequestOptions
{
IfMatchEtag = config.ETag,
})
.ConfigureAwait(false));
}
catch (CosmosException error)
{
if (error.StatusCode == HttpStatusCode.PreconditionFailed)
{
continue;
}
}
}
}
catch (Exception error)
{
TelemetryHelper.Singleton.LogError(
"MigrationConfigDal.CompleteMigrationAsync({0}) failed: {1}",
id,
error);
throw;
}
}
public void AddCurrencies([FromBody] MigrationConfig migrationConfig)
{
var query = db.ReleaseItems.Select(x => x.CurrencyCode).Distinct().ToList();
foreach (var currencyCode in query)
{
if (!String.IsNullOrEmpty(currencyCode))
{
var currency = db.Currencies.Find(currencyCode);
if (currency == null)
{
db.Currencies.Add(new Currency()
{
CurrencyCode = currencyCode
});
}
}
}
db.SaveChanges();
}
public void Execute()
{
var dir = App.BaseMigrationsDirectory + @"\" + _args.Name + @"\";
if (_ds.Exists(dir))
{
throw new GatorException("Warning -- A migration with that name already exists - exiting");
}
_ds.Create(dir);
var cfg = new MigrationConfig {
created = DateTime.Now, versionNumber = "0.0.0"
};
_fs.CreateWithContent(dir + "version.json",
JsonConvert.SerializeObject(cfg, Formatting.Indented, new IsoDateTimeConverter()));
_fs.Create(dir + "up.sql");
_fs.Create(dir + "down.sql");
}
/// <summary>
/// Will be called just after the Schema is created<br/>
/// Can be used to execute custom grants or other database operations.
/// </summary>
/// <param name="expression"></param>
public virtual void ProcessAfter(ICreateSchemaWithPrefixExpression expression)
{
if (!Enabled)
{
return;
}
if (!Processor.SchemaExists(CommonSchemaName))
{
return;
}
Logger.LogInformation($"ProcessAfter Schema:{expression.SchemaName} SchemaPrefix:{expression.SchemaPrefixId}");
var sqlAll = MigrationConfig
.PrepareSql(SqlResources.SchemaPrefixSql)
.ReplaceIgnoreCase("{CommonSchemaName}", CommonSchemaName)
.ReplaceIgnoreCase("{AuthCodeSchemaName}", AuthCodeSchemaName);
Execute(sqlAll);
}
public void Classifications([FromBody] MigrationConfig migrationConfig)
{
Regex adjudicacion = new Regex(@"^adjudicacion-([0-9]+)$", RegexOptions.Compiled);
List <AwaItemsInputDataModel> releaseItemsClassification = this.dataProcessingService.ItemsFrom <AwaItemsInputDataModel>("", "awa_items");
var classifications = releaseItemsClassification
.Where(s => adjudicacion.IsMatch(s.id))
.DistinctBy(s => s.awardsItemsClassificationId)
.Select(y => new ReleaseItemClassification
{
Description = y.awardsItemsClassificationDescription,
ReleaseItemClassificationExternalId = y.awardsItemsClassificationId
});
if (migrationConfig.CheckIfExists)
{
List <ReleaseItemClassification> classificationsToAdd = new List <ReleaseItemClassification>();
foreach (var item in classifications)
{
if (!this.db.ReleaseItemClassifications.Any(x => x.Description == item.Description))
{
classificationsToAdd.Add(item);
}
}
this.db.ReleaseItemClassifications.AddRange(classificationsToAdd);
this.db.SaveChanges();
}
else
{
this.db.ReleaseItemClassifications.AddRange(classifications);
this.db.SaveChanges();
}
}
public void Buyers([FromBody] MigrationConfig migrationConfig)
{
Regex adjudicacion = new Regex(@"^adjudicacion-([0-9]+)$", RegexOptions.Compiled);
List <ReleaseInputDataModel> releasesInput = this.dataProcessingService.ItemsFrom <ReleaseInputDataModel>("", "releases");
var buyers = releasesInput
.Where(s => adjudicacion.IsMatch(s.id))
.DistinctBy(d => d.buyerId)
.Select(y => new Buyer
{
BuyerExternalId = y.buyerId,
Name = y.buyerName
});
if (migrationConfig.CheckIfExists)
{
List <Buyer> buyersToAdd = new List <Buyer>();
foreach (var item in buyers)
{
if (!this.db.Buyers.Any(x => x.Name == item.Name))
{
buyersToAdd.Add(item);
}
}
this.db.Buyers.AddRange(buyersToAdd);
this.db.SaveChanges();
}
else
{
this.db.Buyers.AddRange(buyers);
this.db.SaveChanges();
}
}
public void Suppliers([FromBody] MigrationConfig migrationConfig)
{
Regex adjudicacion = new Regex(@"^adjudicacion-([0-9]+)$", RegexOptions.Compiled);
var suppliersSheet = this.dataProcessingService.ItemsFrom <AwaSuppliersInputDataModel>("", "awa_suppliers");
var suppliers = suppliersSheet
.Where(s => adjudicacion.IsMatch(s.id))
.GroupBy(x => x.awardsSuppliersId)
.Select(y => new Supplier
{
Name = y.First().awardsSuppliersName,
ExternalId = y.Key
}).ToList();
if (migrationConfig.CheckIfExists)
{
List <Supplier> suppliersToAdd = new List <Supplier>();
foreach (var item in suppliers)
{
if (!this.db.Suppliers.Any(x => x.Name == item.Name))
{
suppliersToAdd.Add(item);
}
}
this.db.Suppliers.AddRange(suppliersToAdd);
this.db.SaveChanges();
}
else
{
this.db.Suppliers.AddRange(suppliers);
this.db.SaveChanges();
}
}
public void Cleanup()
{
this.UnderTest = null;
}
public void Init()
{
this.UnderTest = new MigrationConfig();
}
private static async Task TrackMigrationProgressAsync(
Container migrationContainer,
MigrationConfig migrationConfig,
SemaphoreSlim concurrencySempahore)
{
if (migrationContainer == null)
{
throw new ArgumentNullException(nameof(migrationContainer));
}
if (migrationConfig == null)
{
throw new ArgumentNullException(nameof(migrationConfig));
}
if (concurrencySempahore == null)
{
throw new ArgumentNullException(nameof(concurrencySempahore));
}
try
{
CosmosClient sourceClient = GetOrCreateSourceCosmosClient(migrationConfig.MonitoredAccount);
CosmosClient destinationClient = GetOrCreateSourceCosmosClient(migrationConfig.DestAccount);
Container sourceContainer = sourceClient.GetContainer(
migrationConfig.MonitoredDbName,
migrationConfig.MonitoredCollectionName);
Container destinationContainer = destinationClient.GetContainer(
migrationConfig.DestDbName,
migrationConfig.DestCollectionName);
while (true)
{
MigrationConfig migrationConfigSnapshot = await migrationContainer
.ReadItemAsync <MigrationConfig>(migrationConfig.Id, new PartitionKey(migrationConfig.Id))
.ConfigureAwait(false);
DateTimeOffset now = DateTimeOffset.UtcNow;
long sourceCollectionCount = await GetDoucmentCountAsync(sourceContainer).ConfigureAwait(false);
long currentDestinationCollectionCount = await GetDoucmentCountAsync(destinationContainer)
.ConfigureAwait(false);
double currentPercentage = sourceCollectionCount == 0 ?
100 :
currentDestinationCollectionCount * 100.0 / sourceCollectionCount;
long insertedCount =
currentDestinationCollectionCount - migrationConfigSnapshot.MigratedDocumentCount;
long lastMigrationActivityRecorded = Math.Max(
migrationConfigSnapshot.StatisticsLastMigrationActivityRecordedEpochMs,
migrationConfigSnapshot.StartTimeEpochMs);
long nowEpochMs = now.ToUnixTimeMilliseconds();
double currentRate = insertedCount * 1000.0 / (nowEpochMs - lastMigrationActivityRecorded);
long totalSeconds =
(lastMigrationActivityRecorded - migrationConfigSnapshot.StartTimeEpochMs) / 1000;
double averageRate = totalSeconds > 0 ? currentDestinationCollectionCount / totalSeconds : 0;
long etaMs = averageRate > 0
? (long)((sourceCollectionCount - currentDestinationCollectionCount) * 1000 / averageRate)
: (long)TimeSpan.FromDays(100).TotalMilliseconds;
migrationConfigSnapshot.ExpectedDurationLeft = etaMs;
migrationConfigSnapshot.AvgRate = averageRate;
migrationConfigSnapshot.CurrentRate = currentRate;
migrationConfigSnapshot.SourceCountSnapshot = sourceCollectionCount;
migrationConfigSnapshot.DestinationCountSnapshot = currentDestinationCollectionCount;
migrationConfigSnapshot.PercentageCompleted = currentPercentage;
migrationConfigSnapshot.StatisticsLastUpdatedEpochMs = nowEpochMs;
migrationConfigSnapshot.MigratedDocumentCount = currentDestinationCollectionCount;
if (insertedCount > 0)
{
migrationConfigSnapshot.StatisticsLastMigrationActivityRecordedEpochMs = nowEpochMs;
}
try
{
ItemResponse <MigrationConfig> response = await migrationContainer
.ReplaceItemAsync(
migrationConfigSnapshot,
migrationConfigSnapshot.Id,
new PartitionKey(migrationConfigSnapshot.Id),
new ItemRequestOptions
{
IfMatchEtag = migrationConfigSnapshot.ETag
})
.ConfigureAwait(false);
}
catch (CosmosException error)
{
if (error.StatusCode == System.Net.HttpStatusCode.PreconditionFailed)
{
continue;
}
throw;
}
TelemetryHelper.Singleton.TrackStatistics(
sourceCollectionCount,
currentDestinationCollectionCount,
currentPercentage,
currentRate,
averageRate,
etaMs);
return;
}
}
finally
{
concurrencySempahore.Release();
}
}
public MigrationModel(IOptions <MigrationConfig> config)
{
_config = config.Value;
ValidationService = new ValidationService(this);
}
public void Init()
{
this.UnderTest = new MigrationConfig();
}
public void Cleanup()
{
this.UnderTest = null;
}
public MigrationConfigIdParameter(MigrationConfig config) : this(config.Identity)
{
}
