public ChangeFeedProcessorHost(MigrationConfig config) { this.config = config ?? throw new ArgumentNullException(nameof(config)); this.SourcePartitionKeys = config.SourcePartitionKeys; this.TargetPartitionKey = config.TargetPartitionKey; this.leaseCollectionClient = KeyVaultHelper.Singleton.CreateCosmosClientFromKeyVault( EnvironmentConfig.Singleton.MigrationMetadataCosmosAccountName, Program.MigrationClientUserAgentPrefix, useBulk: false, retryOn429Forever: true); this.sourceCollectionClient = KeyVaultHelper.Singleton.CreateCosmosClientFromKeyVault( config.MonitoredAccount, Program.SourceClientUserAgentPrefix, useBulk: false, retryOn429Forever: true); this.destinationCollectionClient = KeyVaultHelper.Singleton.CreateCosmosClientFromKeyVault( config.DestAccount, Program.DestinationClientUserAgentPrefix, useBulk: true, retryOn429Forever: true); this.deadletterClient = KeyVaultHelper.Singleton.GetBlobContainerClientFromKeyVault( EnvironmentConfig.Singleton.DeadLetterAccountName, config.Id?.ToLowerInvariant().Replace("-", String.Empty)); // Make sure to allow multiple active migrations for the same source container // by creating a unique processor name for every config document this.processorName = config.ProcessorName; }
public async Task <IActionResult> OnPostSubmitAsync( string id, string sourceAccount, string sourceDatabase, string sourceContainer, string sourcePK, string destinationAccount, string destinationDatabase, string destinationContainer, string destinationPK) { MigrationConfig newConfig = new MigrationConfig { Id = id, MonitoredAccount = sourceAccount, MonitoredDbName = sourceDatabase, MonitoredCollectionName = sourceContainer, SourcePartitionKeys = sourcePK, DestAccount = destinationAccount, DestDbName = destinationDatabase, DestCollectionName = destinationContainer, TargetPartitionKey = destinationPK, Completed = false, StartTimeEpochMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; await MigrationConfigDal.Singleton .CreateMigrationAsync(newConfig) .ConfigureAwait(false); return(this.RedirectToPage("/Migrations", null)); }
public async Task batch_get_next_for_migration_all_events() { var entity = Guid.NewGuid(); await _store.Append(Setup.UnversionedCommit(guid: entity)); await _store.Append(Setup.UnversionedCommit(guid: entity)); await _store.Append(Setup.UnversionedCommit("1")); var rm = new MigrationConfig("test"); var rez = _store.GetNextBatch(rm, 0); rez.IsEmpty.Should().BeFalse(); var first = rez.GetNext().Value; first.EntityId.Should().Be(entity); first.TenantId.Should().Be("_"); first.Version.Should().Be(1); var second = rez.GetNext().Value; second.EntityId.Should().Be(entity); second.TenantId.Should().Be("_"); second.Version.Should().Be(2); var third = rez.GetNext().Value; third.EntityId.Should().NotBe(entity); third.TenantId.Should().Be("1"); third.Version.Should().Be(1); rez.GetNext().IsEmpty.Should().BeTrue(); }
public MigrationModel(IOptions <MigrationConfig> config, INotificationImportRepository notificationImportRepository) { _config = config.Value; _notificationImportRepository = notificationImportRepository; ValidationService = new ValidationService(this); }
public async Task <MigrationConfig> CreateMigrationAsync(MigrationConfig config) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (String.IsNullOrWhiteSpace(config.Id)) { config.Id = Guid.NewGuid().ToString("N"); } try { BlobContainerClient deadletterClient = KeyVaultHelper.Singleton.GetBlobContainerClientFromKeyVault( EnvironmentConfig.Singleton.DeadLetterAccountName, config.Id?.ToLowerInvariant().Replace("-", String.Empty)); await deadletterClient .CreateIfNotExistsAsync(PublicAccessType.None) .ConfigureAwait(false); return(await this.container .CreateItemAsync <MigrationConfig>(config, new PartitionKey(config.Id)) .ConfigureAwait(false)); } catch (Exception error) { TelemetryHelper.Singleton.LogError( "MigrationConfigDal.CreateMigrationAsync for document with id {0} failed: {1}", config.Id, error); throw; } }
/// <inheritdoc /> public virtual void MigrationMetadataChanged(IMigrationMetadata migrationMetadata, IExtendedMigrationProcessor extendedMigrationProcessor) { Processor = extendedMigrationProcessor; if (!Enabled) { return; } if (!Processor.SchemaExists(CommonSchemaName) || !Processor.SchemaExists(AuthCodeSchemaName)) { return; } var logonSql = MigrationConfig .PrepareSql(SqlResources.LogonScriptSql) .ReplaceIgnoreCase("{CommonSchemaName}", CommonSchemaName) .ReplaceIgnoreCase("{AuthCodeSchemaName}", AuthCodeSchemaName); Execute(logonSql); if (Processor.SchemaExists(MigrationConfig.Schema)) { ProcessAfter(new CreateSchemaWithPrefixExpression { SchemaName = MigrationConfig.Schema, SchemaPrefixId = MigrationConfig.GetSchemaPrefixId(), SchemaPrefixUniqueId = MigrationConfig.GetSchemaPrefixUniqueId() }); } }
public CommittedEvents GetNextBatch(MigrationConfig config, ProcessedCommitsCount count) { IEnumerable <Commit> all = _commits; if (!config.TenantId.IsNullOrEmpty()) { all = all.Where(d => d.TenantId == config.TenantId); } all = all.OrderBy(d => d.Timestamp); return(new CommittedEvents(all.ToArray())); }
public void ReleasesCalculate([FromBody] MigrationConfig migrationConfig) { Dictionary <string, decimal> currencies = db.Currencies.ToDictionary(currency => currency.CurrencyCode, currency => currency.ConversionFactorUYU); var query = db.Releases.Include(release => release.ReleaseItems); foreach (var release in query) { release.TotalAmountUYU = (int)CalculateTotal(release, currencies); } db.SaveChanges(); }
public override void ExecuteCmdlet() { try { if (string.IsNullOrEmpty(LegacyConfigId)) { var migrationPlanList = StorSimpleClient.GetAllMigrationPlan(); if (migrationPlanList.MigrationPlans.Count == 0) { WriteWarning(Resources.MigrationPlanNoConfigs); } else { foreach (var migrationPlan in migrationPlanList.MigrationPlans) { var migrationPlanConfig = new MigrationConfig(migrationPlan); WriteObject(migrationPlanConfig); } } } else { StorSimpleClient.UpdateMigrationPlanSync(LegacyConfigId); var migrationPlanList = StorSimpleClient.GetMigrationPlan(LegacyConfigId); if (0 >= migrationPlanList.MigrationPlans.Count) { throw new ArgumentException(Resources.MigrationPlanNotFound); } else { MigrationPlan migrationPlan = migrationPlanList.MigrationPlans.First(); if (null != LegacyContainerNames) { var legacyContainerNamesList = LegacyContainerNames.ToList(); migrationPlan.MigrationPlanInfo = migrationPlan.MigrationPlanInfo.ToList().FindAll( plan => legacyContainerNamesList.Contains(plan.DataContainerName, StringComparer.InvariantCultureIgnoreCase)); } var migrationPlanMsg = new MigrationPlanMsg(migrationPlan); WriteObject(migrationPlanMsg); } } } catch (Exception except) { this.HandleException(except); } }
public CommittedEvents GetNextBatch(MigrationConfig config, ProcessedCommitsCount count) { using (var db = _db.Create()) { var all = db.QueryAs(q => q .FromAnonymous(new { Id = 1, TenantId = "", EntityId = Guid.Empty }, new TableName(CommitsTable, Schema)).Where(d => true) .AndIf(() => !config.TenantId.IsNullOrEmpty(), d => d.TenantId == config.TenantId) .OrderBy(d => d.Id) .Limit(config.BatchSize, count.Value) .SelectAll(useAsterisk: true).MapTo <Commit>() ); return(new CommittedEvents(all.ToArray())); } }
public async Task <MigrationConfig> CompleteMigrationAsync(string id) { try { while (true) { MigrationConfig config = await this.GetMigrationAsync(id).ConfigureAwait(false); if (config.Completed) { return(config); } config.Completed = true; try { return(await this.container .ReplaceItemAsync <MigrationConfig>( config, config.Id, new PartitionKey(config.Id), new ItemRequestOptions { IfMatchEtag = config.ETag, }) .ConfigureAwait(false)); } catch (CosmosException error) { if (error.StatusCode == HttpStatusCode.PreconditionFailed) { continue; } } } } catch (Exception error) { TelemetryHelper.Singleton.LogError( "MigrationConfigDal.CompleteMigrationAsync({0}) failed: {1}", id, error); throw; } }
public void AddCurrencies([FromBody] MigrationConfig migrationConfig) { var query = db.ReleaseItems.Select(x => x.CurrencyCode).Distinct().ToList(); foreach (var currencyCode in query) { if (!String.IsNullOrEmpty(currencyCode)) { var currency = db.Currencies.Find(currencyCode); if (currency == null) { db.Currencies.Add(new Currency() { CurrencyCode = currencyCode }); } } } db.SaveChanges(); }
public void Execute() { var dir = App.BaseMigrationsDirectory + @"\" + _args.Name + @"\"; if (_ds.Exists(dir)) { throw new GatorException("Warning -- A migration with that name already exists - exiting"); } _ds.Create(dir); var cfg = new MigrationConfig { created = DateTime.Now, versionNumber = "0.0.0" }; _fs.CreateWithContent(dir + "version.json", JsonConvert.SerializeObject(cfg, Formatting.Indented, new IsoDateTimeConverter())); _fs.Create(dir + "up.sql"); _fs.Create(dir + "down.sql"); }
/// <summary> /// Will be called just after the Schema is created<br/> /// Can be used to execute custom grants or other database operations. /// </summary> /// <param name="expression"></param> public virtual void ProcessAfter(ICreateSchemaWithPrefixExpression expression) { if (!Enabled) { return; } if (!Processor.SchemaExists(CommonSchemaName)) { return; } Logger.LogInformation($"ProcessAfter Schema:{expression.SchemaName} SchemaPrefix:{expression.SchemaPrefixId}"); var sqlAll = MigrationConfig .PrepareSql(SqlResources.SchemaPrefixSql) .ReplaceIgnoreCase("{CommonSchemaName}", CommonSchemaName) .ReplaceIgnoreCase("{AuthCodeSchemaName}", AuthCodeSchemaName); Execute(sqlAll); }
public void Classifications([FromBody] MigrationConfig migrationConfig) { Regex adjudicacion = new Regex(@"^adjudicacion-([0-9]+)$", RegexOptions.Compiled); List <AwaItemsInputDataModel> releaseItemsClassification = this.dataProcessingService.ItemsFrom <AwaItemsInputDataModel>("", "awa_items"); var classifications = releaseItemsClassification .Where(s => adjudicacion.IsMatch(s.id)) .DistinctBy(s => s.awardsItemsClassificationId) .Select(y => new ReleaseItemClassification { Description = y.awardsItemsClassificationDescription, ReleaseItemClassificationExternalId = y.awardsItemsClassificationId }); if (migrationConfig.CheckIfExists) { List <ReleaseItemClassification> classificationsToAdd = new List <ReleaseItemClassification>(); foreach (var item in classifications) { if (!this.db.ReleaseItemClassifications.Any(x => x.Description == item.Description)) { classificationsToAdd.Add(item); } } this.db.ReleaseItemClassifications.AddRange(classificationsToAdd); this.db.SaveChanges(); } else { this.db.ReleaseItemClassifications.AddRange(classifications); this.db.SaveChanges(); } }
public void Buyers([FromBody] MigrationConfig migrationConfig) { Regex adjudicacion = new Regex(@"^adjudicacion-([0-9]+)$", RegexOptions.Compiled); List <ReleaseInputDataModel> releasesInput = this.dataProcessingService.ItemsFrom <ReleaseInputDataModel>("", "releases"); var buyers = releasesInput .Where(s => adjudicacion.IsMatch(s.id)) .DistinctBy(d => d.buyerId) .Select(y => new Buyer { BuyerExternalId = y.buyerId, Name = y.buyerName }); if (migrationConfig.CheckIfExists) { List <Buyer> buyersToAdd = new List <Buyer>(); foreach (var item in buyers) { if (!this.db.Buyers.Any(x => x.Name == item.Name)) { buyersToAdd.Add(item); } } this.db.Buyers.AddRange(buyersToAdd); this.db.SaveChanges(); } else { this.db.Buyers.AddRange(buyers); this.db.SaveChanges(); } }
public void Suppliers([FromBody] MigrationConfig migrationConfig) { Regex adjudicacion = new Regex(@"^adjudicacion-([0-9]+)$", RegexOptions.Compiled); var suppliersSheet = this.dataProcessingService.ItemsFrom <AwaSuppliersInputDataModel>("", "awa_suppliers"); var suppliers = suppliersSheet .Where(s => adjudicacion.IsMatch(s.id)) .GroupBy(x => x.awardsSuppliersId) .Select(y => new Supplier { Name = y.First().awardsSuppliersName, ExternalId = y.Key }).ToList(); if (migrationConfig.CheckIfExists) { List <Supplier> suppliersToAdd = new List <Supplier>(); foreach (var item in suppliers) { if (!this.db.Suppliers.Any(x => x.Name == item.Name)) { suppliersToAdd.Add(item); } } this.db.Suppliers.AddRange(suppliersToAdd); this.db.SaveChanges(); } else { this.db.Suppliers.AddRange(suppliers); this.db.SaveChanges(); } }
public void Cleanup() { this.UnderTest = null; }
public void Init() { this.UnderTest = new MigrationConfig(); }
private static async Task TrackMigrationProgressAsync( Container migrationContainer, MigrationConfig migrationConfig, SemaphoreSlim concurrencySempahore) { if (migrationContainer == null) { throw new ArgumentNullException(nameof(migrationContainer)); } if (migrationConfig == null) { throw new ArgumentNullException(nameof(migrationConfig)); } if (concurrencySempahore == null) { throw new ArgumentNullException(nameof(concurrencySempahore)); } try { CosmosClient sourceClient = GetOrCreateSourceCosmosClient(migrationConfig.MonitoredAccount); CosmosClient destinationClient = GetOrCreateSourceCosmosClient(migrationConfig.DestAccount); Container sourceContainer = sourceClient.GetContainer( migrationConfig.MonitoredDbName, migrationConfig.MonitoredCollectionName); Container destinationContainer = destinationClient.GetContainer( migrationConfig.DestDbName, migrationConfig.DestCollectionName); while (true) { MigrationConfig migrationConfigSnapshot = await migrationContainer .ReadItemAsync <MigrationConfig>(migrationConfig.Id, new PartitionKey(migrationConfig.Id)) .ConfigureAwait(false); DateTimeOffset now = DateTimeOffset.UtcNow; long sourceCollectionCount = await GetDoucmentCountAsync(sourceContainer).ConfigureAwait(false); long currentDestinationCollectionCount = await GetDoucmentCountAsync(destinationContainer) .ConfigureAwait(false); double currentPercentage = sourceCollectionCount == 0 ? 100 : currentDestinationCollectionCount * 100.0 / sourceCollectionCount; long insertedCount = currentDestinationCollectionCount - migrationConfigSnapshot.MigratedDocumentCount; long lastMigrationActivityRecorded = Math.Max( migrationConfigSnapshot.StatisticsLastMigrationActivityRecordedEpochMs, migrationConfigSnapshot.StartTimeEpochMs); long nowEpochMs = now.ToUnixTimeMilliseconds(); double currentRate = insertedCount * 1000.0 / (nowEpochMs - lastMigrationActivityRecorded); long totalSeconds = (lastMigrationActivityRecorded - migrationConfigSnapshot.StartTimeEpochMs) / 1000; double averageRate = totalSeconds > 0 ? currentDestinationCollectionCount / totalSeconds : 0; long etaMs = averageRate > 0 ? (long)((sourceCollectionCount - currentDestinationCollectionCount) * 1000 / averageRate) : (long)TimeSpan.FromDays(100).TotalMilliseconds; migrationConfigSnapshot.ExpectedDurationLeft = etaMs; migrationConfigSnapshot.AvgRate = averageRate; migrationConfigSnapshot.CurrentRate = currentRate; migrationConfigSnapshot.SourceCountSnapshot = sourceCollectionCount; migrationConfigSnapshot.DestinationCountSnapshot = currentDestinationCollectionCount; migrationConfigSnapshot.PercentageCompleted = currentPercentage; migrationConfigSnapshot.StatisticsLastUpdatedEpochMs = nowEpochMs; migrationConfigSnapshot.MigratedDocumentCount = currentDestinationCollectionCount; if (insertedCount > 0) { migrationConfigSnapshot.StatisticsLastMigrationActivityRecordedEpochMs = nowEpochMs; } try { ItemResponse <MigrationConfig> response = await migrationContainer .ReplaceItemAsync( migrationConfigSnapshot, migrationConfigSnapshot.Id, new PartitionKey(migrationConfigSnapshot.Id), new ItemRequestOptions { IfMatchEtag = migrationConfigSnapshot.ETag }) .ConfigureAwait(false); } catch (CosmosException error) { if (error.StatusCode == System.Net.HttpStatusCode.PreconditionFailed) { continue; } throw; } TelemetryHelper.Singleton.TrackStatistics( sourceCollectionCount, currentDestinationCollectionCount, currentPercentage, currentRate, averageRate, etaMs); return; } } finally { concurrencySempahore.Release(); } }
public MigrationModel(IOptions <MigrationConfig> config) { _config = config.Value; ValidationService = new ValidationService(this); }
public MigrationConfigIdParameter(MigrationConfig config) : this(config.Identity) { }