public void CrmFileDataExporterTarget()
{
ConsoleLogger.LogLevel = 0;
var orgService = ConnectionHelper.GetOrganizationalServiceTarget();
EntityRepository entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor());
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string fetchXMLFolderPath = Path.Combine(folderPath, "TestData\\FetchXmlFolder");
CrmExporterConfig config = new CrmExporterConfig()
{
FetchXMLFolderPath = fetchXMLFolderPath,
BatchSize = 1000,
PageSize = 100,
TopCount = 10000
};
DataCrmStoreReader storeReader = new DataCrmStoreReader(new ConsoleLogger(), entityRepo, config);
DataFileStoreWriter storeWriter = new DataFileStoreWriter(new ConsoleLogger(), $"{DateTime.UtcNow.ToString("yyMMmmss", CultureInfo.InvariantCulture)}testexportTarget", @"TestData");
CrmFileDataExporter fileExporter = new CrmFileDataExporter(new ConsoleLogger(), storeReader, storeWriter);
FluentActions.Invoking(() => fileExporter.MigrateData())
.Should()
.NotThrow();
}
public void LoadExportConfigFile(INotificationService notificationService, TextBox exportConfigTextBox, Dictionary <string, string> inputFilterQuery, Dictionary <string, Dictionary <string, List <string> > > inputLookupMaping)
{
if (!string.IsNullOrWhiteSpace(exportConfigTextBox.Text))
{
try
{
var configFile = CrmExporterConfig.GetConfiguration(exportConfigTextBox.Text);
if (!configFile.CrmMigrationToolSchemaPaths.Any())
{
notificationService.DisplayFeedback("Invalid Export Config File");
exportConfigTextBox.Text = "";
return;
}
inputFilterQuery.Clear();
inputFilterQuery.AddRange(configFile.CrmMigrationToolSchemaFilters);
inputLookupMaping.Clear();
inputLookupMaping.AddRange(configFile.LookupMapping);
notificationService.DisplayFeedback("Filters and Lookup Mappings loaded from Export Config File");
}
catch (Exception ex)
{
notificationService.DisplayFeedback($"Load Correct Export Config file, error:{ex.Message}");
}
}
}
public void ReadBatchDataFromStore()
{
var orgService = ConnectionHelper.GetOrganizationalServiceSource();
var entRepo = new EntityRepository(orgService, retryExecutor);
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string fetchXMLFolderPath = Path.Combine(folderPath, "TestData\\FetchXmlFolder");
CrmExporterConfig config = new CrmExporterConfig()
{
FetchXMLFolderPath = fetchXMLFolderPath
};
DataCrmStoreReader crmStore = new DataCrmStoreReader(new ConsoleLogger(), entRepo, config);
List <EntityWrapper> results = new List <EntityWrapper>();
List <EntityWrapper> batch = crmStore.ReadBatchDataFromStore();
while (batch.Count > 0)
{
results.AddRange(batch);
batch = crmStore.ReadBatchDataFromStore();
}
Assert.IsTrue(results.Count > 0);
}
public CrmFileDataExporterCsv(ILogger logger, IEntityRepository entityRepo, CrmExporterConfig exporterConfig, CrmSchemaConfiguration schemaConfig, CancellationToken token)
: base(
logger,
new DataCrmStoreReader(logger, entityRepo, exporterConfig),
new DataFileStoreWriterCsv(logger, exporterConfig, schemaConfig),
token)
{
}
protected override void ProcessRecord()
{
base.ProcessRecord();
var logger = new CmdletLoggerPS(this, TreatWarningsAsErrors);
try
{
logger.Info("About to start exporting data from Dynamics365");
var manager = new Dynamics365DataManager();
var cancellationTokenSource = new CancellationTokenSource();
var exportConfig = new CrmExporterConfig();
if (!string.IsNullOrWhiteSpace(ConfigFilePath))
{
if (!File.Exists(ConfigFilePath))
{
WriteWarning($"Export config file path does not exist, will be ignored {ConfigFilePath}");
}
else
{
exportConfig = CrmExporterConfig.GetConfiguration(ConfigFilePath);
}
}
PopulateConfiguration(exportConfig);
if (!Directory.Exists(JsonFolderPath))
{
WriteWarning($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
throw new DirectoryNotFoundException($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
}
else
{
foreach (var file in Directory.GetFiles(JsonFolderPath, $"{exportConfig.FilePrefix}_*.csv"))
{
WriteVerbose($"Delete Csv file {file}");
File.Delete(file);
}
foreach (var file in Directory.GetFiles(JsonFolderPath, $"{exportConfig.FilePrefix}_*.json"))
{
WriteVerbose($"Delete Json file {file}");
File.Delete(file);
}
}
manager.StartSingleThreadedExport(exportConfig, logger, cancellationTokenSource, ConnectionString);
logger.Info("Export has finished");
}
catch (Exception exception)
{
var errorMessage = $"Dynamics365 data export failed: {exception.Message}";
logger.Verbose(errorMessage);
logger.Error(errorMessage);
throw;
}
}
public async Task StartExport(CrmExporterConfig exportConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString)
{
var exportTask = Task.Run(() =>
{
StartSingleThreadedExport(exportConfig, logger, cancellationToken, connectionString);
});
await exportTask;
}
private void InjectAdditionalValuesIntoTheExportConfig(CrmExporterConfig config, ExportSettings exportSettings)
{
config.CrmMigrationToolSchemaPaths.Clear();
config.CrmMigrationToolSchemaPaths.Add(exportSettings.SchemaPath);
config.JsonFolderPath = exportSettings.SavePath;
config.OnlyActiveRecords = !exportSettings.ExportInactiveRecords;
config.BatchSize = exportSettings.BatchSize;
}
public void StartSingleThreadedExport(CrmExporterConfig exportConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString)
{
var connectionHelper = new ConnectionHelper();
var orgService = connectionHelper.GetOrganizationalService(connectionString);
logger.Info("Connectd to instance " + connectionString);
var entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor());
var fileExporter = new CrmFileDataExporter(logger, entityRepo, exportConfig, cancellationToken.Token);
fileExporter.MigrateData();
}
private static int ExportData(ExportOptions options)
{
using (var serviceClient = new CrmServiceClient(options.ConnectionString))
{
new CrmFileDataExporter(
Logger,
new EntityRepository(serviceClient.OrganizationServiceProxy, new ServiceRetryExecutor()),
CrmExporterConfig.GetConfiguration(options.ConfigurationFile),
CancellationToken.None).MigrateData();
return(0);
}
}
public void CrmExporterConfigGetFetchXMLQueries2Test()
{
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string configPath = Path.Combine(folderPath, "TestData\\ImportSchemas\\TestDataSchema\\usersettingsschema.xml");
CrmExporterConfig config = new CrmExporterConfig();
config.CrmMigrationToolSchemaPaths.Add(configPath);
List <string> fetchXmls = config.GetFetchXMLQueries();
Assert.IsTrue(fetchXmls.Count > 0);
}
public void RequestUnknownMigrator()
{
var logger = new Mock <ILogger>().Object;
var entityRepo = new Mock <IEntityRepository>().Object;
var exportConfig = new CrmExporterConfig();
var cancellationToken = CancellationToken.None;
var schema = new CrmSchemaConfiguration();
FluentActions.Invoking(() => systemUnderTest.GetCrmDataMigrator(DataFormat.Unknown, logger, entityRepo, exportConfig, cancellationToken, schema))
.Should()
.Throw <NotSupportedException>();
}
public void CrmExporterConfigGetFetchXMLQueries1Test()
{
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string fetchXMLFolderPath = Path.Combine(folderPath, "TestData\\ImportSchemas\\TestDataSchema");
CrmExporterConfig config = new CrmExporterConfig()
{
FetchXMLFolderPath = fetchXMLFolderPath
};
List <string> fetchXmls = config.GetFetchXMLQueries();
Assert.IsTrue(fetchXmls.Count > 0);
}
public void GetConfiguration()
{
var filePath = "TestData/ExportConfig.json";
var actual = CrmExporterConfig.GetConfiguration(filePath);
actual.PageSize.Should().Be(500);
actual.BatchSize.Should().Be(500);
actual.TopCount.Should().Be(100000);
actual.OnlyActiveRecords.Should().Be(false);
actual.JsonFolderPath.Should().EndWith("TestData");
actual.OneEntityPerBatch.Should().Be(true);
actual.FilePrefix.Should().Be("ExportedData");
actual.SeperateFilesPerEntity.Should().Be(true);
}
public void RequestJsonMigrator()
{
var logger = new Mock <ILogger>().Object;
var entityRepo = new Mock <IEntityRepository>().Object;
var exportConfig = new CrmExporterConfig
{
JsonFolderPath = Path.Combine(Environment.CurrentDirectory, "temp")
};
var cancellationToken = CancellationToken.None;
var schema = new CrmSchemaConfiguration();
var migrator = systemUnderTest.GetCrmDataMigrator(DataFormat.Json, logger, entityRepo, exportConfig, cancellationToken, schema);
migrator.Should().BeOfType <CrmFileDataExporter>();
}
public void CrmFileDataExporterTest3()
{
CrmExporterConfig crmExporterConfig = new CrmExporterConfig
{
PageSize = 50,
BatchSize = 50,
TopCount = 100,
FilePrefix = "Test",
JsonFolderPath = "TestData"
};
FluentActions.Invoking(() => new CrmFileDataExporter(MockLogger.Object, MockEntityRepo.Object, crmExporterConfig, CancellationToken.None))
.Should()
.NotThrow();
}
public static List <RecordCountModel> ExecuteRecordsCount(string exportConfigFilePath, string schemaFilePath, IOrganizationService service, BackgroundWorker worker, DataGridView gridView)
{
worker.ThrowArgumentNullExceptionIfNull(nameof(worker));
gridView.ThrowArgumentNullExceptionIfNull(nameof(gridView));
service.ThrowArgumentNullExceptionIfNull(nameof(service));
CrmExporterConfig crmExporterConfig = CrmExporterConfig.GetConfiguration(exportConfigFilePath);
CrmSchemaConfiguration crmSchemaConfiguration = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
List <RecordCountModel> entityWrapperList = new List <RecordCountModel>();
foreach (var item in crmSchemaConfiguration.Entities)
{
string fetchXml = FetchXmlTemplate;
string filters = crmExporterConfig.CrmMigrationToolSchemaFilters != null && crmExporterConfig.CrmMigrationToolSchemaFilters.ContainsKey(item.Name) ? crmExporterConfig.CrmMigrationToolSchemaFilters[item.Name] : string.Empty;
fetchXml = !string.IsNullOrEmpty(filters) ? fetchXml.Replace("{filter}", filters) : fetchXml.Replace("{filter}", string.Empty);
fetchXml = fetchXml.Replace("{entity}", item.Name);
// Convert the FetchXML into a query expression.
var conversionRequest = new FetchXmlToQueryExpressionRequest
{
FetchXml = fetchXml
};
var conversionResponse =
(FetchXmlToQueryExpressionResponse)service.Execute(conversionRequest);
QueryExpression queryExpression = conversionResponse.Query;
queryExpression.ColumnSet = new ColumnSet(false);
worker.ReportProgress(0, $"Counting... {item.Name}");
var results = service.GetDataByQuery(queryExpression, 5000, false).TotalRecordCount;
worker.ReportProgress(0, $"{item.Name} record count: {results}");
entityWrapperList.Add(new RecordCountModel {
EntityName = item.Name, RecordCount = results
});
gridView.DataSource = null;
gridView.Refresh();
gridView.DataSource = entityWrapperList;
gridView.Columns[0].Width = 250;
gridView.Columns[1].Width = 250;
}
return(entityWrapperList);
}
protected virtual CrmExporterConfig GetExporterConfig()
{
string schemaPath = SchemaPathFull;
string extractedDataPath = ExtractedDataPathFull;
CrmExporterConfig config = new CrmExporterConfig()
{
BatchSize = 500,
PageSize = 500,
TopCount = 100000,
JsonFolderPath = extractedDataPath,
OnlyActiveRecords = false,
OneEntityPerBatch = true
};
config.CrmMigrationToolSchemaPaths.Add(schemaPath);
return(config);
}
static CrmExporterConfig GetExportConfig()
{
var exportConfig = new CrmExporterConfig()
{
BatchSize = 1000,
PageSize = 500,
FilePrefix = $"Demo{Settings.Default.DemoScenarioName}",
OneEntityPerBatch = true,
SeperateFilesPerEntity = true,
TopCount = 10000,
JsonFolderPath = GetExportPath(),
CrmMigrationToolSchemaPaths = new List <string>()
{
GetSchemaPath()
}
};
if (Settings.Default.DemoScenarioName == "Obfuscation")
{
exportConfig.FieldsToObfuscate = GenerateObfuscationConfig();
}
var filePath = $"{GetScenarioPath()}\\ExportConfig.json";
if (!File.Exists(filePath))
{
exportConfig.SaveConfiguration(filePath);
}
else
{
exportConfig = CrmExporterConfig.GetConfiguration(filePath);
}
exportConfig.JsonFolderPath = GetExportPath();
exportConfig.CrmMigrationToolSchemaPaths = new List <string>()
{
GetSchemaPath()
};
return(exportConfig);
}
public void ExportData(ExportSettings exportSettings)
{
if (exportSettings is null)
{
throw new ArgumentNullException(nameof(exportSettings));
}
var tokenSource = new CancellationTokenSource();
var repo = new EntityRepository(exportSettings.EnvironmentConnection, new ServiceRetryExecutor());
if (!string.IsNullOrEmpty(exportSettings.ExportConfigPath))
{
exportConfig = CrmExporterConfig.GetConfiguration(exportSettings.ExportConfigPath);
InjectAdditionalValuesIntoTheExportConfig(exportConfig, exportSettings);
}
else
{
exportConfig = new CrmExporterConfig
{
BatchSize = Convert.ToInt32(exportSettings.BatchSize),
PageSize = 5000,
TopCount = Convert.ToInt32(1000000),
OnlyActiveRecords = !exportSettings.ExportInactiveRecords,
JsonFolderPath = exportSettings.SavePath,
OneEntityPerBatch = true,
FilePrefix = "ExtractedData",
SeperateFilesPerEntity = true,
FetchXMLFolderPath = string.Empty
};
exportConfig.CrmMigrationToolSchemaPaths.Clear();
exportConfig.CrmMigrationToolSchemaPaths.Add(exportSettings.SchemaPath);
}
var schema = CrmSchemaConfiguration.ReadFromFile(exportSettings.SchemaPath);
var exporter = migratorFactory.GetCrmDataMigrator(exportSettings.DataFormat, logger, repo, exportConfig, tokenSource.Token, schema);
exporter.MigrateData();
}
public void CrmExporterConfigSaveTest()
{
var exportConfig = $@"TestData/TempExportConfig.json";
DeleteFileIfItExists(exportConfig);
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string fetchXMLFolderPath = Path.Combine(folderPath, "ImportSchemas\\TestDataSchema");
CrmExporterConfig config = new CrmExporterConfig()
{
FetchXMLFolderPath = fetchXMLFolderPath
};
config.CrmMigrationToolSchemaFilters.Add("entity1", "filter1");
config.CrmMigrationToolSchemaFilters.Add("entity2", "filter2");
config.CrmMigrationToolSchemaFilters.Add("entity3", "filter3");
config.SaveConfiguration(exportConfig);
Assert.IsTrue(File.Exists(exportConfig));
}
private void PopulateConfiguration(CrmExporterConfig exportConfig)
{
exportConfig.CrmMigrationToolSchemaPaths = new List <string> {
SchemaFilePath
};
exportConfig.BatchSize = BatchSize;
exportConfig.PageSize = PageSize;
exportConfig.TopCount = TopCount;
exportConfig.OnlyActiveRecords = OnlyActive;
exportConfig.JsonFolderPath = JsonFolderPath;
WriteVerbose("Export Configuration:");
WriteVerbose($"JsonFolderPath:{exportConfig.JsonFolderPath}");
WriteVerbose($"FilePrefix:{exportConfig.FilePrefix}");
WriteVerbose($"BatchSize:{exportConfig.BatchSize}");
WriteVerbose($"PageSize:{exportConfig.PageSize}");
WriteVerbose($"TopCount:{exportConfig.TopCount}");
WriteVerbose($"OneEntityPerBatch:{exportConfig.OneEntityPerBatch}");
WriteVerbose($"OnlyActiveRecords:{exportConfig.OnlyActiveRecords}");
WriteVerbose($"SeperateFilesPerEntity:{exportConfig.SeperateFilesPerEntity}");
if (exportConfig.ExcludedFields != null && exportConfig.ExcludedFields.Count > 0)
{
WriteVerbose("ExcludedFields:" + string.Join(",", exportConfig.ExcludedFields.ToArray()));
}
if (exportConfig.LookupMapping != null && exportConfig.LookupMapping.Count > 0)
{
foreach (var mapping in exportConfig.LookupMapping)
{
WriteVerbose($"LookupMapping: {mapping.Key}");
foreach (var mapItem in mapping.Value)
{
WriteVerbose($"MapItemKey: {mapItem.Key}");
WriteVerbose($"MapItemValue:{string.Join(",", mapItem.Value.ToArray())}");
}
}
}
}
public void GenerateExportConfigFile(TextBox exportConfig, TextBox schemaPath, Dictionary <string, string> inputFilterQuery, Dictionary <string, Dictionary <string, List <string> > > inputLookupMaping, INotificationService notificationService)
{
try {
var config = new CrmExporterConfig()
{
JsonFolderPath = "ExtractedData",
};
if (File.Exists(exportConfig.Text))
{
config = CrmExporterConfig.GetConfiguration(exportConfig.Text);
}
config.CrmMigrationToolSchemaFilters.Clear();
config.CrmMigrationToolSchemaFilters.AddRange(inputFilterQuery);
if (!string.IsNullOrWhiteSpace(schemaPath.Text))
{
config.CrmMigrationToolSchemaPaths.Clear();
config.CrmMigrationToolSchemaPaths.Add(schemaPath.Text);
}
if (inputLookupMaping.Count > 0)
{
config.LookupMapping.Clear();
config.LookupMapping.AddRange(inputLookupMaping);
}
if (File.Exists(exportConfig.Text))
{
File.Delete(exportConfig.Text);
}
config.SaveConfiguration(exportConfig.Text);
}
catch (Exception ex)
{
notificationService.DisplayFeedback($"Error Saving Export Config file. Error: {ex.Message}");
}
}
public void Setup()
{
InitializeProperties();
systemUnderTest = new CrmExporterConfig();
}
public GenericCrmDataMigrator GetCrmDataMigrator(DataFormat dataFormat, ILogger logger, IEntityRepository repo, CrmExporterConfig exportConfig, CancellationToken token, CrmSchemaConfiguration schema)
{
switch (dataFormat)
{
case DataFormat.Json:
return(new CrmFileDataExporter(logger, repo, exportConfig, token));
case DataFormat.Csv:
return(new CrmFileDataExporterCsv(logger, repo, exportConfig, schema, token));
default:
throw new NotSupportedException($"Data format: '{dataFormat}' is not supported.");
}
}
public void CrmExporterConfigReadTest()
{
var mgr = CrmExporterConfig.GetConfiguration(@"TestData/ExportConfig.json");
Assert.AreEqual("C:\\GitRepos\\UserSettings\\usersettingsschema.xml", mgr.CrmMigrationToolSchemaPaths[0]);
}