private static void StartMultiThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
logger.Info($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
var cnt = Convert.ToInt32(maxThreads);
while (cnt > 0)
{
cnt--;
repos.Add(new EntityRepository(connectionHelper.GetOrganizationalService(connectionString), new ServiceRetryExecutor()));
logger.Info("New connection created to " + connectionString);
}
CrmGenericImporter fileImporter = null;
if (!useCsv)
{
fileImporter = new CrmFileDataImporter(logger, repos, importConfig, cancellationToken.Token);
}
else
{
fileImporter = new CrmFileDataImporterCsv(logger, repos, importConfig, schemaConfig, cancellationToken.Token);
}
fileImporter.MigrateData();
}
private static CrmImportConfig GetImportConfig(DataImportConfig dataImportConfig, string packageFolderPath)
{
var importConfig = CrmImportConfig.GetConfiguration(Path.Combine(packageFolderPath, dataImportConfig.ImportConfigPath));
importConfig.JsonFolderPath = Path.Combine(packageFolderPath, dataImportConfig.DataFolderPath);
return(importConfig);
}
public void LoadImportConfigFile(INotificationService notificationService, TextBox importConfig, Dictionary <string, Dictionary <Guid, Guid> > inputMapper, Dictionary <string, List <Item <EntityReference, EntityReference> > > inputMapping)
{
if (!string.IsNullOrWhiteSpace(importConfig.Text))
{
try
{
var configImport = CrmImportConfig.GetConfiguration(importConfig.Text);
if (configImport.MigrationConfig == null)
{
notificationService.DisplayFeedback("Invalid Import Config File");
importConfig.Text = "";
return;
}
inputMapper = configImport.MigrationConfig.Mappings;
DataConversion(inputMapping, inputMapper);
notificationService.DisplayFeedback("Guid Id Mappings loaded from Import Config File");
}
catch (Exception ex)
{
notificationService.DisplayFeedback($"Load Correct Import Config file, error:{ex.Message}");
}
}
}
public void GetConfiguration()
{
var filePath = "TestData/ImportConfig.json";
var actual = CrmImportConfig.GetConfiguration(filePath);
actual.FilePrefix.Should().Be("ExportedData");
}
public CrmFileDataImporter(ILogger logger, IEntityRepository entityRepo, CrmImportConfig importConfig, CancellationToken token)
: base(
logger,
new DataFileStoreReader(logger, importConfig),
new DataCrmStoreWriter(logger, entityRepo, importConfig, token),
importConfig,
token)
{
}
private void TabImportConfigFileTextChanged(object sender, EventArgs e)
{
importConfig = CrmImportConfig.GetConfiguration(openFileDialog1.FileName);
cbIgnoreSystemFields.Checked = importConfig.IgnoreSystemFields;
cbIgnoreStatuses.Checked = importConfig.IgnoreStatuses;
tbSourceDataLocation.Text = importConfig.JsonFolderPath;
nudSavePageSize.Value = importConfig.SaveBatchSize;
}
public void Setup()
{
InitializeProperties();
mockReaderConfig = new Mock <IFileStoreReaderConfig>();
MockCrmStoreWriterConfig = new Mock <ICrmStoreWriterConfig>();
mockCrmGenericImporterConfig = new Mock <ICrmGenericImporterConfig>();
schemaConfig = new CrmSchemaConfiguration();
crmImportConfig = new CrmImportConfig();
}
private static int ImportData(ImportOptions options)
{
using (var serviceClient = new CrmServiceClient(options.ConnectionString))
{
new CrmFileDataImporter(
Logger,
new EntityRepository(serviceClient.OrganizationServiceProxy, new ServiceRetryExecutor()),
CrmImportConfig.GetConfiguration(options.ConfigurationFile),
CancellationToken.None).MigrateData();
return(0);
}
}
public void CrmImportProcessorConfigSaveTest()
{
CrmImportConfig mgr = new CrmImportConfig
{
MigrationConfig = new MappingConfiguration()
};
var guidMap = new Dictionary <Guid, Guid>
{
{ Guid.Parse("4ee93159-38c1-e611-80e9-c4346bacbdc4"), Guid.Empty },
{ Guid.Parse("e47e726c-24bd-e611-80e6-c4346bad4198"), Guid.Empty },
{ Guid.Parse("75c28436-3ec1-e611-80e9-c4346bacbdc4"), Guid.Empty }
};
mgr.MigrationConfig.Mappings.Add("test1", guidMap);
var guidMap2 = new Dictionary <Guid, Guid>
{
{ Guid.Parse("018b25db-37c1-e611-80e9-c4346bacbdc4"), Guid.Empty },
{ Guid.Parse("e8a524d4-37c1-e611-80e9-c4346bacbdc4"), Guid.Empty },
{ Guid.Parse("c11b9c7b-3ac1-e611-80e9-c4346bacbdc4"), Guid.Empty },
{ Guid.Parse("c8144a2d-50bd-e611-80e6-c4346bad4198"), Guid.Empty }
};
mgr.MigrationConfig.Mappings.Add("test2", guidMap2);
mgr.AdditionalFieldsToIgnore.AddRange(new List <string> {
"TEST1", "Test2"
});
mgr.EntitiesToSync.AddRange(new List <string> {
"TESTEnt1", "TestEnt2"
});
mgr.JsonFolderPath = @"C:\xxxx\xxx.json";
mgr.PluginsToDeactivate.AddRange(new List <Tuple <string, string> > {
new Tuple <string, string>("plugin name", "assembly nme"), new Tuple <string, string>("plugin name2", "assembly nme2")
});
mgr.ProcessesToDeactivate.AddRange(new List <string> {
"test", "test2"
});
mgr.IgnoreStatuses = true;
mgr.IgnoreStatusesExceptions.AddRange(new List <string>()
{
"ent1", "ent2"
});
mgr.SaveConfiguration(ImportConfigExample);
Assert.IsTrue(File.Exists(ImportConfigExample));
}
public void CrmFileDataImporterConstructor4()
{
var importConfig = new CrmImportConfig
{
FilePrefix = "Import",
JsonFolderPath = "TestData",
SaveBatchSize = 500
};
FluentActions.Invoking(() => new CrmFileDataImporter(MockLogger.Object, MockEntityRepo.Object, importConfig, CancellationToken.None))
.Should()
.NotThrow();
}
protected virtual CrmImportConfig GetImporterConfig()
{
CrmImportConfig importConfig = new CrmImportConfig()
{
IgnoreStatuses = true,
IgnoreSystemFields = true,
JsonFolderPath = ExtractedDataPathFull,
SaveBatchSize = 500,
MigrationConfig = new MappingConfiguration()
};
return(importConfig);
}
public void TestSetup()
{
logger = new Mock <ILogger>();
entityRepositoryService = new Mock <IEntityRepositoryService>();
importConfig = new CrmImportConfig()
{
IgnoreStatuses = false,
IgnoreSystemFields = true,
SaveBatchSize = 5000,
JsonFolderPath = "TestData",
FilePrefix = "ExtractedData"
};
}
private static int ImportData(ImportOptions options)
{
using (var serviceClient = new CrmServiceClient(options.ConnectionString))
{
var config = CrmImportConfig.GetConfiguration(options.ConfigurationFile);
new CrmFileDataImporter(
Logger,
new EntityRepository(serviceClient, new ServiceRetryExecutor(), config.EnableLookupCaching ? RepositoryCachingMode.Lookup : RepositoryCachingMode.None),
config,
CancellationToken.None).MigrateData();
return(0);
}
}
public async Task StartImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var importTask = Task.Run(() =>
{
logger.Info("Connectd to instance " + connectionString);
if (maxThreads > 1 && !string.IsNullOrWhiteSpace(connectionString))
{
StartMultiThreadedImport(importConfig, logger, cancellationToken, connectionString, maxThreads, useCsv, schemaConfig);
}
else
{
StartSingleThreadedImport(importConfig, logger, cancellationToken, connectionString, useCsv, schemaConfig);
}
});
await importTask;
}
public ImportWizard()
{
InitializeComponent();
importConfig = new CrmImportConfig()
{
IgnoreStatuses = cbIgnoreStatuses.Checked,
IgnoreSystemFields = cbIgnoreSystemFields.Checked,
SaveBatchSize = Convert.ToInt32(nudSavePageSize.Value),
JsonFolderPath = tbSourceDataLocation.Text,
FilePrefix = "ExtractedData"
};
wizardButtons1.OnExecute += Button2Click;
logger = new LoggerService(tbLogger, SynchronizationContext.Current);
wizardButtons1.OnCustomNextNavigation += WizardButtons1OnNavigateToNextPage;
}
public void StartSingleThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
var orgService = connectionHelper.GetOrganizationalService(connectionString);
logger.Info("Starting Single Threaded processing, you must configure connection string for multithreaded processing adn set up max threads to more than 1");
var entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor());
CrmGenericImporter fileImporter = null;
if (!useCsv)
{
fileImporter = new CrmFileDataImporter(logger, entityRepo, importConfig, cancellationToken.Token);
}
else
{
fileImporter = new CrmFileDataImporterCsv(logger, entityRepo, importConfig, schemaConfig, cancellationToken.Token);
}
fileImporter.MigrateData();
}
public void GenerateImportConfigFile(INotificationService notificationService, TextBox importConfig, Dictionary <string, Dictionary <Guid, Guid> > inputMapper)
{
try
{
var migration = new CrmImportConfig()
{
IgnoreStatuses = true,
IgnoreSystemFields = true,
SaveBatchSize = 1000,
JsonFolderPath = "ExtractedData"
};
if (File.Exists(importConfig.Text))
{
migration = CrmImportConfig.GetConfiguration(importConfig.Text);
}
if (migration.MigrationConfig == null)
{
migration.MigrationConfig = new MappingConfiguration();
}
if (inputMapper != null)
{
migration.MigrationConfig.Mappings.Clear();
migration.MigrationConfig.Mappings.AddRange(inputMapper);
if (File.Exists(importConfig.Text))
{
File.Delete(importConfig.Text);
}
migration.SaveConfiguration(importConfig.Text);
}
}
catch (Exception ex)
{
notificationService.DisplayFeedback($"Error Saving Import Config file. Error: {ex.Message}");
}
}
public void CrmFileDataImporterWithIgnoredFieldsTest()
{
ConsoleLogger.LogLevel = 3;
using (CancellationTokenSource tokenSource = new CancellationTokenSource())
{
List <IEntityRepository> entRep = new List <IEntityRepository>
{
new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()),
new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()),
new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor()),
new EntityRepository(ConnectionHelper.GetOrganizationalServiceTarget(), new ServiceRetryExecutor())
};
CrmImportConfig importConfig = CrmImportConfig.GetConfiguration(@"TestData\ImportConfig.json");
CrmFileDataImporter fileExporter = new CrmFileDataImporter(new ConsoleLogger(), entRep, importConfig, tokenSource.Token);
fileExporter.MigrateData();
Assert.IsNotNull(fileExporter);
}
}
static CrmImportConfig GetImportConfig()
{
var importConfig = new CrmImportConfig()
{
FilePrefix = $"Demo{Settings.Default.DemoScenarioName}",
SaveBatchSize = 50,
JsonFolderPath = GetExportPath(),
};
var filePath = $"{GetScenarioPath()}\\ImportConfig.json";
if (!File.Exists(filePath))
{
importConfig.SaveConfiguration(filePath);
}
else
{
importConfig = CrmImportConfig.GetConfiguration(filePath);
}
importConfig.JsonFolderPath = GetExportPath();
return(importConfig);
}
public CrmFileDataImporterCsv(ILogger logger, List <IEntityRepository> entityRepos, CrmImportConfig importConfig, CrmSchemaConfiguration schemaConfig, CancellationToken token)
: base(
logger,
new DataFileStoreReaderCsv(logger, importConfig, schemaConfig),
new DataCrmStoreWriterMultiThreaded(logger, entityRepos, importConfig, token),
importConfig,
token)
{
}
public void CrmImportProcessorConfigReadTest()
{
var mgr = CrmImportConfig.GetConfiguration(@"TestData/ImportConfig.json");
Assert.IsNotNull(mgr);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
var logger = new CmdletLogger(TreatWarningsAsErrors);
try
{
logger.LogInfo("About to start importing data from Dynamics365");
var manager = new Dynamics365DataManager();
var cancellationTokenSource = new CancellationTokenSource();
var importConfig = new CrmImportConfig();
if (!string.IsNullOrWhiteSpace(ConfigFilePath))
{
if (!File.Exists(ConfigFilePath))
{
WriteWarning($"Import config file path does not exist, will be ignored {ConfigFilePath}");
}
else
{
importConfig = CrmImportConfig.GetConfiguration(ConfigFilePath);
}
}
PopulateConfigFile(importConfig);
if (!Directory.Exists(JsonFolderPath))
{
WriteWarning($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
throw new DirectoryNotFoundException($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
}
CrmSchemaConfiguration schemaConfig = null;
if (CsvImport)
{
if (string.IsNullOrWhiteSpace(SchemaFilePath))
{
throw new ConfigurationException("Schema file is required for CSV Import!");
}
schemaConfig = CrmSchemaConfiguration.ReadFromFile(SchemaFilePath);
logger.LogInfo("Using Csv import");
}
else
{
logger.LogInfo("Using JSon import");
}
if (MaxThreads > 1)
{
var result = manager.StartImport(importConfig, logger, cancellationTokenSource, ConnectionString, MaxThreads, CsvImport, schemaConfig)
.ContinueWith(a =>
{
logger.LogInfo("Dynamics365 data import completed successfully.");
},
cancellationTokenSource.Token);
result.Wait(cancellationTokenSource.Token);
}
else
{
manager.StartSingleThreadedImport(importConfig, new CmdletLoggerPS(this, TreatWarningsAsErrors), cancellationTokenSource, ConnectionString, CsvImport, schemaConfig);
}
}
catch (Exception exception)
{
var errorMessage = $"Dynamics365 data import failed: {exception.Message}";
logger.LogVerbose(errorMessage);
logger.LogError(errorMessage);
throw;
}
}
private void PopulateConfigFile(CrmImportConfig importConfig)
{
importConfig.IgnoreStatuses = IgnoreStatuses;
importConfig.IgnoreSystemFields = IgnoreSystemFields;
importConfig.SaveBatchSize = SavePageSize;
importConfig.JsonFolderPath = JsonFolderPath;
WriteVerbose("Import Configuration:");
WriteVerbose("JsonFolderPath:" + importConfig.JsonFolderPath);
WriteVerbose("SaveBatchSize:" + importConfig.SaveBatchSize);
WriteVerbose("IgnoreStatuses:" + importConfig.IgnoreStatuses);
WriteVerbose("IgnoreSystemFields:" + importConfig.IgnoreSystemFields);
WriteVerbose("DeactivateAllProcesses:" + importConfig.DeactivateAllProcesses);
if (importConfig.PluginsToDeactivate != null && importConfig.PluginsToDeactivate.Count > 0)
{
WriteVerbose("PluginsToDeactivate:" + string.Join(",", importConfig.PluginsToDeactivate.Select(p => p.Item1 + ":" + p.Item2).ToArray()));
}
if (importConfig.ProcessesToDeactivate != null && importConfig.ProcessesToDeactivate.Count > 0)
{
WriteVerbose("ProcessesToDeactivate:" + string.Join(",", importConfig.ProcessesToDeactivate.ToArray()));
}
if (importConfig.EntitiesToSync != null && importConfig.EntitiesToSync.Count > 0)
{
WriteVerbose("EntitiesToSync:" + string.Join(",", importConfig.EntitiesToSync.ToArray()));
}
if (importConfig.FiledsToIgnore != null && importConfig.FiledsToIgnore.Count > 0)
{
WriteVerbose("FiledsToIgnore:" + string.Join(",", importConfig.FiledsToIgnore.ToArray()));
}
if (importConfig.NoUpsertEntities != null && importConfig.NoUpsertEntities.Count > 0)
{
WriteVerbose("NoUpsertEntities:" + string.Join(",", importConfig.NoUpsertEntities.ToArray()));
}
if (importConfig.IgnoreStatusesExceptions != null && importConfig.IgnoreStatusesExceptions.Count > 0)
{
WriteVerbose("IgnoreStatusesExceptions:" + string.Join(",", importConfig.IgnoreStatusesExceptions.ToArray()));
}
if (importConfig.MigrationConfig != null)
{
WriteVerbose($"MigrationConfig:");
WriteVerbose($"SourceRootBUName: {importConfig.MigrationConfig.SourceRootBUName}");
WriteVerbose("ApplyAliasMapping:" + importConfig.MigrationConfig.ApplyAliasMapping);
if (importConfig.MigrationConfig.Mappings != null)
{
foreach (var item in importConfig.MigrationConfig.Mappings)
{
WriteVerbose("Maping for entity:" + item.Key);
foreach (var map in item.Value)
{
WriteVerbose("SourceId:" + map.Key + ", TargetId:" + map.Value);
}
}
}
}
}
public void Setup()
{
InitializeProperties();
systemUnderTest = new CrmImportConfig();
}
public void PerformImportAction(string importSchemaFilePath, int maxThreads, bool jsonFormat, Capgemini.DataMigration.Core.ILogger currentLogger, IEntityRepositoryService entityRepositoryService, CrmImportConfig currentImportConfig, CancellationTokenSource tokenSource)
{
try
{
if (maxThreads > 1)
{
currentLogger.LogInfo($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
int threadCount = maxThreads;
while (threadCount > 0)
{
threadCount--;
repos.Add(entityRepositoryService.InstantiateEntityRepository(true));
}
var fileExporter = new CrmFileDataImporter(currentLogger, repos, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
currentLogger.LogInfo("Starting Single Threaded processing, you must set up max threads to more than 1");
var entityRepo = entityRepositoryService.InstantiateEntityRepository(false);
if (jsonFormat)
{
var fileExporter = new CrmFileDataImporter(currentLogger, entityRepo, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
var schema = CrmSchemaConfiguration.ReadFromFile(importSchemaFilePath);
var fileExporter = new CrmFileDataImporterCsv(currentLogger, entityRepo, currentImportConfig, schema, tokenSource.Token);
fileExporter.MigrateData();
}
}
}
catch (Exception ex)
{
currentLogger.LogError($"Critical import error, processing stopped: {ex.Message}");
throw;
}
}
