private static void StartMultiThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig) { var connectionHelper = new ConnectionHelper(); logger.Info($"Starting MultiThreaded Processing, using {maxThreads} threads"); var repos = new List <IEntityRepository>(); var cnt = Convert.ToInt32(maxThreads); while (cnt > 0) { cnt--; repos.Add(new EntityRepository(connectionHelper.GetOrganizationalService(connectionString), new ServiceRetryExecutor())); logger.Info("New connection created to " + connectionString); } CrmGenericImporter fileImporter = null; if (!useCsv) { fileImporter = new CrmFileDataImporter(logger, repos, importConfig, cancellationToken.Token); } else { fileImporter = new CrmFileDataImporterCsv(logger, repos, importConfig, schemaConfig, cancellationToken.Token); } fileImporter.MigrateData(); }
private static CrmImportConfig GetImportConfig(DataImportConfig dataImportConfig, string packageFolderPath) { var importConfig = CrmImportConfig.GetConfiguration(Path.Combine(packageFolderPath, dataImportConfig.ImportConfigPath)); importConfig.JsonFolderPath = Path.Combine(packageFolderPath, dataImportConfig.DataFolderPath); return(importConfig); }
public void LoadImportConfigFile(INotificationService notificationService, TextBox importConfig, Dictionary <string, Dictionary <Guid, Guid> > inputMapper, Dictionary <string, List <Item <EntityReference, EntityReference> > > inputMapping) { if (!string.IsNullOrWhiteSpace(importConfig.Text)) { try { var configImport = CrmImportConfig.GetConfiguration(importConfig.Text); if (configImport.MigrationConfig == null) { notificationService.DisplayFeedback("Invalid Import Config File"); importConfig.Text = ""; return; } inputMapper = configImport.MigrationConfig.Mappings; DataConversion(inputMapping, inputMapper); notificationService.DisplayFeedback("Guid Id Mappings loaded from Import Config File"); } catch (Exception ex) { notificationService.DisplayFeedback($"Load Correct Import Config file, error:{ex.Message}"); } } }
public void GetConfiguration() { var filePath = "TestData/ImportConfig.json"; var actual = CrmImportConfig.GetConfiguration(filePath); actual.FilePrefix.Should().Be("ExportedData"); }
public CrmFileDataImporter(ILogger logger, IEntityRepository entityRepo, CrmImportConfig importConfig, CancellationToken token) : base( logger, new DataFileStoreReader(logger, importConfig), new DataCrmStoreWriter(logger, entityRepo, importConfig, token), importConfig, token) { }
private void TabImportConfigFileTextChanged(object sender, EventArgs e) { importConfig = CrmImportConfig.GetConfiguration(openFileDialog1.FileName); cbIgnoreSystemFields.Checked = importConfig.IgnoreSystemFields; cbIgnoreStatuses.Checked = importConfig.IgnoreStatuses; tbSourceDataLocation.Text = importConfig.JsonFolderPath; nudSavePageSize.Value = importConfig.SaveBatchSize; }
public void Setup() { InitializeProperties(); mockReaderConfig = new Mock <IFileStoreReaderConfig>(); MockCrmStoreWriterConfig = new Mock <ICrmStoreWriterConfig>(); mockCrmGenericImporterConfig = new Mock <ICrmGenericImporterConfig>(); schemaConfig = new CrmSchemaConfiguration(); crmImportConfig = new CrmImportConfig(); }
private static int ImportData(ImportOptions options) { using (var serviceClient = new CrmServiceClient(options.ConnectionString)) { new CrmFileDataImporter( Logger, new EntityRepository(serviceClient.OrganizationServiceProxy, new ServiceRetryExecutor()), CrmImportConfig.GetConfiguration(options.ConfigurationFile), CancellationToken.None).MigrateData(); return(0); } }
public void CrmImportProcessorConfigSaveTest() { CrmImportConfig mgr = new CrmImportConfig { MigrationConfig = new MappingConfiguration() }; var guidMap = new Dictionary <Guid, Guid> { { Guid.Parse("4ee93159-38c1-e611-80e9-c4346bacbdc4"), Guid.Empty }, { Guid.Parse("e47e726c-24bd-e611-80e6-c4346bad4198"), Guid.Empty }, { Guid.Parse("75c28436-3ec1-e611-80e9-c4346bacbdc4"), Guid.Empty } }; mgr.MigrationConfig.Mappings.Add("test1", guidMap); var guidMap2 = new Dictionary <Guid, Guid> { { Guid.Parse("018b25db-37c1-e611-80e9-c4346bacbdc4"), Guid.Empty }, { Guid.Parse("e8a524d4-37c1-e611-80e9-c4346bacbdc4"), Guid.Empty }, { Guid.Parse("c11b9c7b-3ac1-e611-80e9-c4346bacbdc4"), Guid.Empty }, { Guid.Parse("c8144a2d-50bd-e611-80e6-c4346bad4198"), Guid.Empty } }; mgr.MigrationConfig.Mappings.Add("test2", guidMap2); mgr.AdditionalFieldsToIgnore.AddRange(new List <string> { "TEST1", "Test2" }); mgr.EntitiesToSync.AddRange(new List <string> { "TESTEnt1", "TestEnt2" }); mgr.JsonFolderPath = @"C:\xxxx\xxx.json"; mgr.PluginsToDeactivate.AddRange(new List <Tuple <string, string> > { new Tuple <string, string>("plugin name", "assembly nme"), new Tuple <string, string>("plugin name2", "assembly nme2") }); mgr.ProcessesToDeactivate.AddRange(new List <string> { "test", "test2" }); mgr.IgnoreStatuses = true; mgr.IgnoreStatusesExceptions.AddRange(new List <string>() { "ent1", "ent2" }); mgr.SaveConfiguration(ImportConfigExample); Assert.IsTrue(File.Exists(ImportConfigExample)); }
public void CrmFileDataImporterConstructor4() { var importConfig = new CrmImportConfig { FilePrefix = "Import", JsonFolderPath = "TestData", SaveBatchSize = 500 }; FluentActions.Invoking(() => new CrmFileDataImporter(MockLogger.Object, MockEntityRepo.Object, importConfig, CancellationToken.None)) .Should() .NotThrow(); }
protected virtual CrmImportConfig GetImporterConfig() { CrmImportConfig importConfig = new CrmImportConfig() { IgnoreStatuses = true, IgnoreSystemFields = true, JsonFolderPath = ExtractedDataPathFull, SaveBatchSize = 500, MigrationConfig = new MappingConfiguration() }; return(importConfig); }
public void TestSetup() { logger = new Mock <ILogger>(); entityRepositoryService = new Mock <IEntityRepositoryService>(); importConfig = new CrmImportConfig() { IgnoreStatuses = false, IgnoreSystemFields = true, SaveBatchSize = 5000, JsonFolderPath = "TestData", FilePrefix = "ExtractedData" }; }
private static int ImportData(ImportOptions options) { using (var serviceClient = new CrmServiceClient(options.ConnectionString)) { var config = CrmImportConfig.GetConfiguration(options.ConfigurationFile); new CrmFileDataImporter( Logger, new EntityRepository(serviceClient, new ServiceRetryExecutor(), config.EnableLookupCaching ? RepositoryCachingMode.Lookup : RepositoryCachingMode.None), config, CancellationToken.None).MigrateData(); return(0); } }
public async Task StartImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig) { var importTask = Task.Run(() => { logger.Info("Connectd to instance " + connectionString); if (maxThreads > 1 && !string.IsNullOrWhiteSpace(connectionString)) { StartMultiThreadedImport(importConfig, logger, cancellationToken, connectionString, maxThreads, useCsv, schemaConfig); } else { StartSingleThreadedImport(importConfig, logger, cancellationToken, connectionString, useCsv, schemaConfig); } }); await importTask; }
public ImportWizard() { InitializeComponent(); importConfig = new CrmImportConfig() { IgnoreStatuses = cbIgnoreStatuses.Checked, IgnoreSystemFields = cbIgnoreSystemFields.Checked, SaveBatchSize = Convert.ToInt32(nudSavePageSize.Value), JsonFolderPath = tbSourceDataLocation.Text, FilePrefix = "ExtractedData" }; wizardButtons1.OnExecute += Button2Click; logger = new LoggerService(tbLogger, SynchronizationContext.Current); wizardButtons1.OnCustomNextNavigation += WizardButtons1OnNavigateToNextPage; }
public void StartSingleThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, bool useCsv, CrmSchemaConfiguration schemaConfig) { var connectionHelper = new ConnectionHelper(); var orgService = connectionHelper.GetOrganizationalService(connectionString); logger.Info("Starting Single Threaded processing, you must configure connection string for multithreaded processing adn set up max threads to more than 1"); var entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor()); CrmGenericImporter fileImporter = null; if (!useCsv) { fileImporter = new CrmFileDataImporter(logger, entityRepo, importConfig, cancellationToken.Token); } else { fileImporter = new CrmFileDataImporterCsv(logger, entityRepo, importConfig, schemaConfig, cancellationToken.Token); } fileImporter.MigrateData(); }
public void GenerateImportConfigFile(INotificationService notificationService, TextBox importConfig, Dictionary <string, Dictionary <Guid, Guid> > inputMapper) { try { var migration = new CrmImportConfig() { IgnoreStatuses = true, IgnoreSystemFields = true, SaveBatchSize = 1000, JsonFolderPath = "ExtractedData" }; if (File.Exists(importConfig.Text)) { migration = CrmImportConfig.GetConfiguration(importConfig.Text); } if (migration.MigrationConfig == null) { migration.MigrationConfig = new MappingConfiguration(); } if (inputMapper != null) { migration.MigrationConfig.Mappings.Clear(); migration.MigrationConfig.Mappings.AddRange(inputMapper); if (File.Exists(importConfig.Text)) { File.Delete(importConfig.Text); } migration.SaveConfiguration(importConfig.Text); } } catch (Exception ex) { notificationService.DisplayFeedback($"Error Saving Import Config file. Error: {ex.Message}"); } }
public void CrmFileDataImporterWithIgnoredFieldsTest() { ConsoleLogger.LogLevel = 3; using (CancellationTokenSource tokenSource = new CancellationTokenSource()) { List <IEntityRepository> entRep = new List <IEntityRepository> { new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()), new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()), new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()), new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()) }; CrmImportConfig importConfig = CrmImportConfig.GetConfiguration(@"TestData\ImportConfig.json"); CrmFileDataImporter fileExporter = new CrmFileDataImporter(new ConsoleLogger(), entRep, importConfig, tokenSource.Token); fileExporter.MigrateData(); Assert.IsNotNull(fileExporter); } }
static CrmImportConfig GetImportConfig() { var importConfig = new CrmImportConfig() { FilePrefix = $"Demo{Settings.Default.DemoScenarioName}", SaveBatchSize = 50, JsonFolderPath = GetExportPath(), }; var filePath = $"{GetScenarioPath()}\\ImportConfig.json"; if (!File.Exists(filePath)) { importConfig.SaveConfiguration(filePath); } else { importConfig = CrmImportConfig.GetConfiguration(filePath); } importConfig.JsonFolderPath = GetExportPath(); return(importConfig); }
public CrmFileDataImporterCsv(ILogger logger, List <IEntityRepository> entityRepos, CrmImportConfig importConfig, CrmSchemaConfiguration schemaConfig, CancellationToken token) : base( logger, new DataFileStoreReaderCsv(logger, importConfig, schemaConfig), new DataCrmStoreWriterMultiThreaded(logger, entityRepos, importConfig, token), importConfig, token) { }
public void CrmImportProcessorConfigReadTest() { var mgr = CrmImportConfig.GetConfiguration(@"TestData/ImportConfig.json"); Assert.IsNotNull(mgr); }
protected override void ProcessRecord() { base.ProcessRecord(); var logger = new CmdletLogger(TreatWarningsAsErrors); try { logger.LogInfo("About to start importing data from Dynamics365"); var manager = new Dynamics365DataManager(); var cancellationTokenSource = new CancellationTokenSource(); var importConfig = new CrmImportConfig(); if (!string.IsNullOrWhiteSpace(ConfigFilePath)) { if (!File.Exists(ConfigFilePath)) { WriteWarning($"Import config file path does not exist, will be ignored {ConfigFilePath}"); } else { importConfig = CrmImportConfig.GetConfiguration(ConfigFilePath); } } PopulateConfigFile(importConfig); if (!Directory.Exists(JsonFolderPath)) { WriteWarning($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!"); throw new DirectoryNotFoundException($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!"); } CrmSchemaConfiguration schemaConfig = null; if (CsvImport) { if (string.IsNullOrWhiteSpace(SchemaFilePath)) { throw new ConfigurationException("Schema file is required for CSV Import!"); } schemaConfig = CrmSchemaConfiguration.ReadFromFile(SchemaFilePath); logger.LogInfo("Using Csv import"); } else { logger.LogInfo("Using JSon import"); } if (MaxThreads > 1) { var result = manager.StartImport(importConfig, logger, cancellationTokenSource, ConnectionString, MaxThreads, CsvImport, schemaConfig) .ContinueWith(a => { logger.LogInfo("Dynamics365 data import completed successfully."); }, cancellationTokenSource.Token); result.Wait(cancellationTokenSource.Token); } else { manager.StartSingleThreadedImport(importConfig, new CmdletLoggerPS(this, TreatWarningsAsErrors), cancellationTokenSource, ConnectionString, CsvImport, schemaConfig); } } catch (Exception exception) { var errorMessage = $"Dynamics365 data import failed: {exception.Message}"; logger.LogVerbose(errorMessage); logger.LogError(errorMessage); throw; } }
private void PopulateConfigFile(CrmImportConfig importConfig) { importConfig.IgnoreStatuses = IgnoreStatuses; importConfig.IgnoreSystemFields = IgnoreSystemFields; importConfig.SaveBatchSize = SavePageSize; importConfig.JsonFolderPath = JsonFolderPath; WriteVerbose("Import Configuration:"); WriteVerbose("JsonFolderPath:" + importConfig.JsonFolderPath); WriteVerbose("SaveBatchSize:" + importConfig.SaveBatchSize); WriteVerbose("IgnoreStatuses:" + importConfig.IgnoreStatuses); WriteVerbose("IgnoreSystemFields:" + importConfig.IgnoreSystemFields); WriteVerbose("DeactivateAllProcesses:" + importConfig.DeactivateAllProcesses); if (importConfig.PluginsToDeactivate != null && importConfig.PluginsToDeactivate.Count > 0) { WriteVerbose("PluginsToDeactivate:" + string.Join(",", importConfig.PluginsToDeactivate.Select(p => p.Item1 + ":" + p.Item2).ToArray())); } if (importConfig.ProcessesToDeactivate != null && importConfig.ProcessesToDeactivate.Count > 0) { WriteVerbose("ProcessesToDeactivate:" + string.Join(",", importConfig.ProcessesToDeactivate.ToArray())); } if (importConfig.EntitiesToSync != null && importConfig.EntitiesToSync.Count > 0) { WriteVerbose("EntitiesToSync:" + string.Join(",", importConfig.EntitiesToSync.ToArray())); } if (importConfig.FiledsToIgnore != null && importConfig.FiledsToIgnore.Count > 0) { WriteVerbose("FiledsToIgnore:" + string.Join(",", importConfig.FiledsToIgnore.ToArray())); } if (importConfig.NoUpsertEntities != null && importConfig.NoUpsertEntities.Count > 0) { WriteVerbose("NoUpsertEntities:" + string.Join(",", importConfig.NoUpsertEntities.ToArray())); } if (importConfig.IgnoreStatusesExceptions != null && importConfig.IgnoreStatusesExceptions.Count > 0) { WriteVerbose("IgnoreStatusesExceptions:" + string.Join(",", importConfig.IgnoreStatusesExceptions.ToArray())); } if (importConfig.MigrationConfig != null) { WriteVerbose($"MigrationConfig:"); WriteVerbose($"SourceRootBUName: {importConfig.MigrationConfig.SourceRootBUName}"); WriteVerbose("ApplyAliasMapping:" + importConfig.MigrationConfig.ApplyAliasMapping); if (importConfig.MigrationConfig.Mappings != null) { foreach (var item in importConfig.MigrationConfig.Mappings) { WriteVerbose("Maping for entity:" + item.Key); foreach (var map in item.Value) { WriteVerbose("SourceId:" + map.Key + ", TargetId:" + map.Value); } } } } }
public void Setup() { InitializeProperties(); systemUnderTest = new CrmImportConfig(); }
public void PerformImportAction(string importSchemaFilePath, int maxThreads, bool jsonFormat, Capgemini.DataMigration.Core.ILogger currentLogger, IEntityRepositoryService entityRepositoryService, CrmImportConfig currentImportConfig, CancellationTokenSource tokenSource) { try { if (maxThreads > 1) { currentLogger.LogInfo($"Starting MultiThreaded Processing, using {maxThreads} threads"); var repos = new List <IEntityRepository>(); int threadCount = maxThreads; while (threadCount > 0) { threadCount--; repos.Add(entityRepositoryService.InstantiateEntityRepository(true)); } var fileExporter = new CrmFileDataImporter(currentLogger, repos, currentImportConfig, tokenSource.Token); fileExporter.MigrateData(); } else { currentLogger.LogInfo("Starting Single Threaded processing, you must set up max threads to more than 1"); var entityRepo = entityRepositoryService.InstantiateEntityRepository(false); if (jsonFormat) { var fileExporter = new CrmFileDataImporter(currentLogger, entityRepo, currentImportConfig, tokenSource.Token); fileExporter.MigrateData(); } else { var schema = CrmSchemaConfiguration.ReadFromFile(importSchemaFilePath); var fileExporter = new CrmFileDataImporterCsv(currentLogger, entityRepo, currentImportConfig, schema, tokenSource.Token); fileExporter.MigrateData(); } } } catch (Exception ex) { currentLogger.LogError($"Critical import error, processing stopped: {ex.Message}"); throw; } }