/// <summary>
/// Generates FetchXMLQueries.
/// </summary>
/// <returns>value.</returns>
public List <string> GetFetchXMLQueries()
{
List <string> result = new List <string>();
if (!string.IsNullOrEmpty(FetchXMLFolderPath))
{
var dir = new DirectoryInfo(FetchXMLFolderPath);
var files = dir.GetFiles("*.xml");
foreach (var file in files)
{
result.Add(File.ReadAllText(file.FullName));
}
}
if (CrmMigrationToolSchemaPaths != null && CrmMigrationToolSchemaPaths.Count > 0)
{
foreach (var schemaPath in CrmMigrationToolSchemaPaths)
{
CrmSchemaConfiguration schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
result.AddRange(schema.PrepareFetchXMLFromSchema(OnlyActiveRecords, CrmMigrationToolSchemaFilters, GetMappingFetchCreators()));
}
}
return(result);
}
public static void ImportData(string connectionString, string schemaPath, string exportFolderPath)
{
Console.WriteLine("Import Started");
var tokenSource = new CancellationTokenSource();
var serviceClient = new CrmServiceClient(connectionString);
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor());
var logger = new ConsoleLogger();
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entityRepo, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
//Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entityRepo, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
Console.WriteLine("Import Finished");
}
private static void ExportData(string connectionString, string schemaPath)
{
Console.WriteLine("Export Started");
using (var tokenSource = new CancellationTokenSource())
{
using (var serviceClient = new CrmServiceClient(connectionString))
{
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor(), RepositoryCachingMode.Lookup);
var logger = new ConsoleLogger();
if (!Settings.Default.UseCsvImport)
{
// Json Export
var fileExporterJson = new CrmFileDataExporter(logger, entityRepo, GetExportConfig(), tokenSource.Token);
fileExporterJson.MigrateData();
}
else
{
// Csv Export
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileExporterCsv = new CrmFileDataExporterCsv(logger, entityRepo, GetExportConfig(), schema, tokenSource.Token);
fileExporterCsv.MigrateData();
}
}
}
Console.WriteLine("Export Finished");
}
public void CrmSchemaConfigReadTest()
{
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string configPath = Path.Combine(folderPath, "TestData\\ImportSchemas\\TestDataSchema\\usersettingsschema.xml");
CrmSchemaConfiguration config = CrmSchemaConfiguration.ReadFromFile(configPath);
Assert.IsNotNull(config);
}
private string DataImport(bool useCsv)
{
ConsoleLogger.LogLevel = 5;
var logger = new ConsoleLogger();
List <IEntityRepository> entityRepos = new List <IEntityRepository>();
int cnt = 0;
while (entityRepos.Count < threadsNo)
{
if (useFakeRepo)
{
entityRepos.Add(new EntityRepositoryFake(targetServices[cnt], new ServiceRetryExecutor()));
}
else
{
entityRepos.Add(new EntityRepository(targetServices[cnt], new ServiceRetryExecutor()));
}
cnt++;
}
var importConfig = GetImporterConfig();
string importConfigPath = Path.Combine(SchemaFolderPathFull, ConfigPrefix + "ImportConfig.json");
if (File.Exists(importConfigPath))
{
File.Delete(importConfigPath);
}
importConfig.SaveConfiguration(importConfigPath);
CrmGenericImporter fileImporter;
if (useCsv)
{
var schema = CrmSchemaConfiguration.ReadFromFile(SchemaPathFull);
fileImporter = new CrmFileDataImporterCsv(logger, entityRepos, importConfig, schema, new CancellationToken(false));
}
else
{
fileImporter = new CrmFileDataImporter(logger, entityRepos, importConfig, new CancellationToken(false));
}
var watch = Stopwatch.StartNew();
fileImporter.MigrateData();
watch.Stop();
var message = $"Importing Data completed, took {watch.Elapsed.TotalSeconds} seconds";
Debug.WriteLine(message);
return(message);
}
public void LoadSchemaFile(string schemaFilePath, bool working, INotificationService notificationService, Dictionary <string, HashSet <string> > inputEntityAttributes, Dictionary <string, HashSet <string> > inputEntityRelationships)
{
if (!string.IsNullOrWhiteSpace(schemaFilePath))
{
try
{
var crmSchema = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
var controller = new EntityController();
controller.StoreEntityData(crmSchema.Entities?.ToArray(), inputEntityAttributes, inputEntityRelationships);
ClearAllListViews();
PopulateEntities(working);
}
catch (Exception ex)
{
notificationService.DisplayFeedback($"Schema File load error, ensure to load correct Schema file, Error: {ex.Message}");
}
}
}
public static List <RecordCountModel> ExecuteRecordsCount(string exportConfigFilePath, string schemaFilePath, IOrganizationService service, BackgroundWorker worker, DataGridView gridView)
{
worker.ThrowArgumentNullExceptionIfNull(nameof(worker));
gridView.ThrowArgumentNullExceptionIfNull(nameof(gridView));
service.ThrowArgumentNullExceptionIfNull(nameof(service));
CrmExporterConfig crmExporterConfig = CrmExporterConfig.GetConfiguration(exportConfigFilePath);
CrmSchemaConfiguration crmSchemaConfiguration = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
List <RecordCountModel> entityWrapperList = new List <RecordCountModel>();
foreach (var item in crmSchemaConfiguration.Entities)
{
string fetchXml = FetchXmlTemplate;
string filters = crmExporterConfig.CrmMigrationToolSchemaFilters != null && crmExporterConfig.CrmMigrationToolSchemaFilters.ContainsKey(item.Name) ? crmExporterConfig.CrmMigrationToolSchemaFilters[item.Name] : string.Empty;
fetchXml = !string.IsNullOrEmpty(filters) ? fetchXml.Replace("{filter}", filters) : fetchXml.Replace("{filter}", string.Empty);
fetchXml = fetchXml.Replace("{entity}", item.Name);
// Convert the FetchXML into a query expression.
var conversionRequest = new FetchXmlToQueryExpressionRequest
{
FetchXml = fetchXml
};
var conversionResponse =
(FetchXmlToQueryExpressionResponse)service.Execute(conversionRequest);
QueryExpression queryExpression = conversionResponse.Query;
queryExpression.ColumnSet = new ColumnSet(false);
worker.ReportProgress(0, $"Counting... {item.Name}");
var results = service.GetDataByQuery(queryExpression, 5000, false).TotalRecordCount;
worker.ReportProgress(0, $"{item.Name} record count: {results}");
entityWrapperList.Add(new RecordCountModel {
EntityName = item.Name, RecordCount = results
});
gridView.DataSource = null;
gridView.Refresh();
gridView.DataSource = entityWrapperList;
gridView.Columns[0].Width = 250;
gridView.Columns[1].Width = 250;
}
return(entityWrapperList);
}
public void PerformImportAction(string importSchemaFilePath, int maxThreads, bool jsonFormat, Capgemini.DataMigration.Core.ILogger currentLogger, IEntityRepositoryService entityRepositoryService, CrmImportConfig currentImportConfig, CancellationTokenSource tokenSource)
{
try
{
if (maxThreads > 1)
{
currentLogger.LogInfo($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
int threadCount = maxThreads;
while (threadCount > 0)
{
threadCount--;
repos.Add(entityRepositoryService.InstantiateEntityRepository(true));
}
var fileExporter = new CrmFileDataImporter(currentLogger, repos, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
currentLogger.LogInfo("Starting Single Threaded processing, you must set up max threads to more than 1");
var entityRepo = entityRepositoryService.InstantiateEntityRepository(false);
if (jsonFormat)
{
var fileExporter = new CrmFileDataImporter(currentLogger, entityRepo, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
var schema = CrmSchemaConfiguration.ReadFromFile(importSchemaFilePath);
var fileExporter = new CrmFileDataImporterCsv(currentLogger, entityRepo, currentImportConfig, schema, tokenSource.Token);
fileExporter.MigrateData();
}
}
}
catch (Exception ex)
{
currentLogger.LogError($"Critical import error, processing stopped: {ex.Message}");
throw;
}
}
public void DataExportTest()
{
ConsoleLogger.LogLevel = 5;
var orgService = SourceService;
var repo = new EntityRepository(orgService, new ServiceRetryExecutor());
var config = GetExporterConfig();
string exportConfigPath = Path.Combine(SchemaFolderPathFull, $"{ConfigPrefix}ExportConfig.json");
if (File.Exists(exportConfigPath))
{
File.Delete(exportConfigPath);
}
config.SaveConfiguration(exportConfigPath);
if (Directory.Exists(ExtractedDataPathFull))
{
Directory.Delete(ExtractedDataPathFull, true);
}
Directory.CreateDirectory(ExtractedDataPathFull);
CrmFileDataExporter fileExporter = new CrmFileDataExporter(new ConsoleLogger(), repo, config, new CancellationToken(false));
FluentActions.Invoking(() => fileExporter.MigrateData())
.Should()
.NotThrow();
if (useCSV)
{
var schema = CrmSchemaConfiguration.ReadFromFile(SchemaPathFull);
CrmFileDataExporterCsv fileExporterCsv = new CrmFileDataExporterCsv(new ConsoleLogger(), repo, config, schema, new CancellationToken(false));
FluentActions.Invoking(() => fileExporterCsv.MigrateData())
.Should()
.NotThrow();
}
}
private Tuple <List <Entity>, List <Entity> > GetFirstEntities(string filePrefix)
{
string extractedPath = Path.Combine(TestBase.GetWorkiongFolderPath(), "TestData");
string extractFolder = Path.Combine(extractedPath, "ExtractedData");
string schemaFilePath = Path.Combine(extractedPath, "usersettingsschema.xml");
CrmSchemaConfiguration schemaConfig = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
DataFileStoreReaderCsv store = new DataFileStoreReaderCsv(new ConsoleLogger(), filePrefix, extractFolder, schemaConfig);
var batch = store.ReadBatchDataFromStore();
List <Entity> firstEnt = batch.Select(p => p.OriginalEntity).ToList();
DataFileStoreReader storeJson = new DataFileStoreReader(new ConsoleLogger(), filePrefix, extractFolder);
var batchJson = storeJson.ReadBatchDataFromStore();
List <Entity> firstEntJson = batchJson.Select(p => p.OriginalEntity).ToList();
return(new Tuple <List <Entity>, List <Entity> >(firstEnt, firstEntJson));
}
public void ExportData(ExportSettings exportSettings)
{
if (exportSettings is null)
{
throw new ArgumentNullException(nameof(exportSettings));
}
var tokenSource = new CancellationTokenSource();
var repo = new EntityRepository(exportSettings.EnvironmentConnection, new ServiceRetryExecutor());
if (!string.IsNullOrEmpty(exportSettings.ExportConfigPath))
{
exportConfig = CrmExporterConfig.GetConfiguration(exportSettings.ExportConfigPath);
InjectAdditionalValuesIntoTheExportConfig(exportConfig, exportSettings);
}
else
{
exportConfig = new CrmExporterConfig
{
BatchSize = Convert.ToInt32(exportSettings.BatchSize),
PageSize = 5000,
TopCount = Convert.ToInt32(1000000),
OnlyActiveRecords = !exportSettings.ExportInactiveRecords,
JsonFolderPath = exportSettings.SavePath,
OneEntityPerBatch = true,
FilePrefix = "ExtractedData",
SeperateFilesPerEntity = true,
FetchXMLFolderPath = string.Empty
};
exportConfig.CrmMigrationToolSchemaPaths.Clear();
exportConfig.CrmMigrationToolSchemaPaths.Add(exportSettings.SchemaPath);
}
var schema = CrmSchemaConfiguration.ReadFromFile(exportSettings.SchemaPath);
var exporter = migratorFactory.GetCrmDataMigrator(exportSettings.DataFormat, logger, repo, exportConfig, tokenSource.Token, schema);
exporter.MigrateData();
}
protected override void ProcessRecord()
{
base.ProcessRecord();
var logger = new CmdletLogger(TreatWarningsAsErrors);
try
{
logger.LogInfo("About to start importing data from Dynamics365");
var manager = new Dynamics365DataManager();
var cancellationTokenSource = new CancellationTokenSource();
var importConfig = new CrmImportConfig();
if (!string.IsNullOrWhiteSpace(ConfigFilePath))
{
if (!File.Exists(ConfigFilePath))
{
WriteWarning($"Import config file path does not exist, will be ignored {ConfigFilePath}");
}
else
{
importConfig = CrmImportConfig.GetConfiguration(ConfigFilePath);
}
}
PopulateConfigFile(importConfig);
if (!Directory.Exists(JsonFolderPath))
{
WriteWarning($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
throw new DirectoryNotFoundException($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
}
CrmSchemaConfiguration schemaConfig = null;
if (CsvImport)
{
if (string.IsNullOrWhiteSpace(SchemaFilePath))
{
throw new ConfigurationException("Schema file is required for CSV Import!");
}
schemaConfig = CrmSchemaConfiguration.ReadFromFile(SchemaFilePath);
logger.LogInfo("Using Csv import");
}
else
{
logger.LogInfo("Using JSon import");
}
if (MaxThreads > 1)
{
var result = manager.StartImport(importConfig, logger, cancellationTokenSource, ConnectionString, MaxThreads, CsvImport, schemaConfig)
.ContinueWith(a =>
{
logger.LogInfo("Dynamics365 data import completed successfully.");
},
cancellationTokenSource.Token);
result.Wait(cancellationTokenSource.Token);
}
else
{
manager.StartSingleThreadedImport(importConfig, new CmdletLoggerPS(this, TreatWarningsAsErrors), cancellationTokenSource, ConnectionString, CsvImport, schemaConfig);
}
}
catch (Exception exception)
{
var errorMessage = $"Dynamics365 data import failed: {exception.Message}";
logger.LogVerbose(errorMessage);
logger.LogError(errorMessage);
throw;
}
}
private static void ImportData(string connectionString, string schemaPath)
{
Console.WriteLine("Import Started");
using (var tokenSource = new CancellationTokenSource())
{
using (var serviceClient = new CrmServiceClient(connectionString))
{
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor());
int threadCount = Settings.Default.ThreadCount;
var logger = new ConsoleLogger();
if (threadCount > 1)
{
List <IEntityRepository> entRepos = new List <IEntityRepository>()
{
entityRepo
};
while (threadCount > 1)
{
threadCount--;
var newServiceClient = serviceClient.Clone();
if (newServiceClient != null)
{
entRepos.Add(new EntityRepository(serviceClient.Clone(), new ServiceRetryExecutor()));
}
else
{
Console.WriteLine("CrmServiceClient Clone() operation is only supported for OAUTH connections, single thread will be used as clone returned null");
}
}
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entRepos, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
// Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entRepos, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
}
else
{
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entityRepo, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
// Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entityRepo, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
}
}
}
Console.WriteLine("Import Finished");
}
public void ReadSchemaFromFile()
{
var schema = CrmSchemaConfiguration.ReadFromFile("TestData/usersettingsschema.xml");
Assert.IsNotNull(schema);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
var logger = new CmdletLoggerPS(this, TreatWarningsAsErrors);
try
{
logger.LogInfo("About to start exporting data from Dynamics365");
var manager = new Dynamics365DataManager();
var cancellationTokenSource = new CancellationTokenSource();
var exportConfig = new CrmExporterConfig();
if (!string.IsNullOrWhiteSpace(ConfigFilePath))
{
if (!File.Exists(ConfigFilePath))
{
WriteWarning($"Export config file path does not exist, will be ignored {ConfigFilePath}");
}
else
{
exportConfig = CrmExporterConfig.GetConfiguration(ConfigFilePath);
}
}
PopulateConfiguration(exportConfig);
if (!Directory.Exists(JsonFolderPath))
{
WriteWarning($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
throw new DirectoryNotFoundException($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
}
else
{
foreach (var file in Directory.GetFiles(JsonFolderPath, $"{exportConfig.FilePrefix}_*.csv"))
{
WriteVerbose($"Delete Csv file {file}");
File.Delete(file);
}
foreach (var file in Directory.GetFiles(JsonFolderPath, $"{exportConfig.FilePrefix}_*.json"))
{
WriteVerbose($"Delete Json file {file}");
File.Delete(file);
}
}
CrmSchemaConfiguration schemaConfig = null;
if (CsvExport)
{
if (string.IsNullOrWhiteSpace(SchemaFilePath))
{
throw new ConfigurationErrorsException("Schema file is required for CSV Export!");
}
schemaConfig = CrmSchemaConfiguration.ReadFromFile(SchemaFilePath);
logger.LogInfo("Using Csv Export");
}
else
{
logger.LogInfo("Using JSon Export");
}
manager.StartSingleThreadedExport(exportConfig, logger, cancellationTokenSource, ConnectionString, CsvExport, schemaConfig);
logger.LogInfo("Export has finished");
}
catch (Exception exception)
{
var errorMessage = $"Dynamics365 data export failed: {exception.Message}";
logger.LogVerbose(errorMessage);
logger.LogError(errorMessage);
throw;
}
}