public static void ImportData(string connectionString, string schemaPath, string exportFolderPath)
{
Console.WriteLine("Import Started");
var tokenSource = new CancellationTokenSource();
var serviceClient = new CrmServiceClient(connectionString);
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor());
var logger = new ConsoleLogger();
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entityRepo, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
//Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entityRepo, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
Console.WriteLine("Import Finished");
}
public void GenerateXMLFile(TextBox tbSchemaPath, CrmSchemaConfiguration schemaConfiguration)
{
if (!string.IsNullOrWhiteSpace(tbSchemaPath.Text))
{
schemaConfiguration.SaveToFile(tbSchemaPath.Text);
}
}
private static void ExportData(string connectionString, string schemaPath)
{
Console.WriteLine("Export Started");
using (var tokenSource = new CancellationTokenSource())
{
using (var serviceClient = new CrmServiceClient(connectionString))
{
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor(), RepositoryCachingMode.Lookup);
var logger = new ConsoleLogger();
if (!Settings.Default.UseCsvImport)
{
// Json Export
var fileExporterJson = new CrmFileDataExporter(logger, entityRepo, GetExportConfig(), tokenSource.Token);
fileExporterJson.MigrateData();
}
else
{
// Csv Export
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileExporterCsv = new CrmFileDataExporterCsv(logger, entityRepo, GetExportConfig(), schema, tokenSource.Token);
fileExporterCsv.MigrateData();
}
}
}
Console.WriteLine("Export Finished");
}
/// <summary>
/// Generates FetchXMLQueries.
/// </summary>
/// <returns>value.</returns>
public List <string> GetFetchXMLQueries()
{
List <string> result = new List <string>();
if (!string.IsNullOrEmpty(FetchXMLFolderPath))
{
var dir = new DirectoryInfo(FetchXMLFolderPath);
var files = dir.GetFiles("*.xml");
foreach (var file in files)
{
result.Add(File.ReadAllText(file.FullName));
}
}
if (CrmMigrationToolSchemaPaths != null && CrmMigrationToolSchemaPaths.Count > 0)
{
foreach (var schemaPath in CrmMigrationToolSchemaPaths)
{
CrmSchemaConfiguration schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
result.AddRange(schema.PrepareFetchXMLFromSchema(OnlyActiveRecords, CrmMigrationToolSchemaFilters, GetMappingFetchCreators()));
}
}
return(result);
}
private static CrmSchemaConfiguration GetSchema()
{
CrmSchemaConfiguration schemaConfig = new CrmSchemaConfiguration();
List <CrmEntity> entities = new List <CrmEntity>();
CrmEntity contact = new CrmEntity
{
Name = "contact",
PrimaryIdField = "contactid"
};
List <CrmField> fields = new List <CrmField>
{
new CrmField {
FieldName = "contactid", FieldType = "guid"
},
new CrmField {
FieldName = "firstname", FieldType = "string"
},
new CrmField {
FieldName = "lastname", FieldType = "string"
}
};
contact.CrmFields.AddRange(fields);
entities.Add(contact);
schemaConfig.Entities.AddRange(entities);
return(schemaConfig);
}
public DataFileStoreReaderCsv(ILogger logger, IFileStoreReaderConfig config, CrmSchemaConfiguration schemaConfig)
: this(
logger,
config?.FilePrefix,
config?.JsonFolderPath,
schemaConfig)
{
}
public void CrmSchemaConfigReadTest()
{
string folderPath = new FileInfo(Assembly.GetExecutingAssembly().Location).DirectoryName;
string configPath = Path.Combine(folderPath, "TestData\\ImportSchemas\\TestDataSchema\\usersettingsschema.xml");
CrmSchemaConfiguration config = CrmSchemaConfiguration.ReadFromFile(configPath);
Assert.IsNotNull(config);
}
public DataFileStoreWriterCsv(ILogger logger, IFileStoreWriterConfig config, CrmSchemaConfiguration schemaConfig)
: this(
logger,
config?.FilePrefix,
config?.JsonFolderPath,
config?.ExcludedFields,
schemaConfig)
{
}
public void Setup()
{
InitializeProperties();
mockReaderConfig = new Mock <IFileStoreReaderConfig>();
MockCrmStoreWriterConfig = new Mock <ICrmStoreWriterConfig>();
mockCrmGenericImporterConfig = new Mock <ICrmGenericImporterConfig>();
schemaConfig = new CrmSchemaConfiguration();
crmImportConfig = new CrmImportConfig();
}
private string DataImport(bool useCsv)
{
ConsoleLogger.LogLevel = 5;
var logger = new ConsoleLogger();
List <IEntityRepository> entityRepos = new List <IEntityRepository>();
int cnt = 0;
while (entityRepos.Count < threadsNo)
{
if (useFakeRepo)
{
entityRepos.Add(new EntityRepositoryFake(targetServices[cnt], new ServiceRetryExecutor()));
}
else
{
entityRepos.Add(new EntityRepository(targetServices[cnt], new ServiceRetryExecutor()));
}
cnt++;
}
var importConfig = GetImporterConfig();
string importConfigPath = Path.Combine(SchemaFolderPathFull, ConfigPrefix + "ImportConfig.json");
if (File.Exists(importConfigPath))
{
File.Delete(importConfigPath);
}
importConfig.SaveConfiguration(importConfigPath);
CrmGenericImporter fileImporter;
if (useCsv)
{
var schema = CrmSchemaConfiguration.ReadFromFile(SchemaPathFull);
fileImporter = new CrmFileDataImporterCsv(logger, entityRepos, importConfig, schema, new CancellationToken(false));
}
else
{
fileImporter = new CrmFileDataImporter(logger, entityRepos, importConfig, new CancellationToken(false));
}
var watch = Stopwatch.StartNew();
fileImporter.MigrateData();
watch.Stop();
var message = $"Importing Data completed, took {watch.Elapsed.TotalSeconds} seconds";
Debug.WriteLine(message);
return(message);
}
public void RequestUnknownMigrator()
{
var logger = new Mock <ILogger>().Object;
var entityRepo = new Mock <IEntityRepository>().Object;
var exportConfig = new CrmExporterConfig();
var cancellationToken = CancellationToken.None;
var schema = new CrmSchemaConfiguration();
FluentActions.Invoking(() => systemUnderTest.GetCrmDataMigrator(DataFormat.Unknown, logger, entityRepo, exportConfig, cancellationToken, schema))
.Should()
.Throw <NotSupportedException>();
}
public void Setup()
{
InitializeProperties();
var entity = new CrmEntity
{
Name = "contact",
PrimaryIdField = "contactid"
};
crmSchemaConfiguration = new CrmSchemaConfiguration();
crmSchemaConfiguration.Entities.Add(entity);
systemUnderTest = new DataFileStoreWriterCsv(MockLogger.Object, FilePrefix, TestResultFolder, ExcludedFields, crmSchemaConfiguration);
}
public void RequestJsonMigrator()
{
var logger = new Mock <ILogger>().Object;
var entityRepo = new Mock <IEntityRepository>().Object;
var exportConfig = new CrmExporterConfig
{
JsonFolderPath = Path.Combine(Environment.CurrentDirectory, "temp")
};
var cancellationToken = CancellationToken.None;
var schema = new CrmSchemaConfiguration();
var migrator = systemUnderTest.GetCrmDataMigrator(DataFormat.Json, logger, entityRepo, exportConfig, cancellationToken, schema);
migrator.Should().BeOfType <CrmFileDataExporter>();
}
public void CollectCrmEntityFields(HashSet <string> inputCheckedEntity, CrmSchemaConfiguration schemaConfiguration, Dictionary <string, HashSet <string> > inputEntityRelationships, Dictionary <string, HashSet <string> > inputEntityAttributes, AttributeTypeMapping inputAttributeMapping, ServiceParameters serviceParameters)
{
if (inputCheckedEntity.Count > 0)
{
var crmEntityList = new List <CrmEntity>();
foreach (var item in inputCheckedEntity)
{
var crmEntity = new CrmEntity();
var sourceList = serviceParameters.MetadataService.RetrieveEntities(item, serviceParameters.OrganizationService, serviceParameters.ExceptionService);
StoreCrmEntityData(crmEntity, sourceList, crmEntityList, inputEntityRelationships, inputEntityAttributes, inputAttributeMapping, serviceParameters.NotificationService);
}
schemaConfiguration.Entities.Clear();
schemaConfiguration.Entities.AddRange(crmEntityList);
}
}
public void LoadSchemaFile(string schemaFilePath, bool working, INotificationService notificationService, Dictionary <string, HashSet <string> > inputEntityAttributes, Dictionary <string, HashSet <string> > inputEntityRelationships)
{
if (!string.IsNullOrWhiteSpace(schemaFilePath))
{
try
{
var crmSchema = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
var controller = new EntityController();
controller.StoreEntityData(crmSchema.Entities?.ToArray(), inputEntityAttributes, inputEntityRelationships);
ClearAllListViews();
PopulateEntities(working);
}
catch (Exception ex)
{
notificationService.DisplayFeedback($"Schema File load error, ensure to load correct Schema file, Error: {ex.Message}");
}
}
}
public void PerformImportAction(string importSchemaFilePath, int maxThreads, bool jsonFormat, Capgemini.DataMigration.Core.ILogger currentLogger, IEntityRepositoryService entityRepositoryService, CrmImportConfig currentImportConfig, CancellationTokenSource tokenSource)
{
try
{
if (maxThreads > 1)
{
currentLogger.LogInfo($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
int threadCount = maxThreads;
while (threadCount > 0)
{
threadCount--;
repos.Add(entityRepositoryService.InstantiateEntityRepository(true));
}
var fileExporter = new CrmFileDataImporter(currentLogger, repos, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
currentLogger.LogInfo("Starting Single Threaded processing, you must set up max threads to more than 1");
var entityRepo = entityRepositoryService.InstantiateEntityRepository(false);
if (jsonFormat)
{
var fileExporter = new CrmFileDataImporter(currentLogger, entityRepo, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
var schema = CrmSchemaConfiguration.ReadFromFile(importSchemaFilePath);
var fileExporter = new CrmFileDataImporterCsv(currentLogger, entityRepo, currentImportConfig, schema, tokenSource.Token);
fileExporter.MigrateData();
}
}
}
catch (Exception ex)
{
currentLogger.LogError($"Critical import error, processing stopped: {ex.Message}");
throw;
}
}
public static List <RecordCountModel> ExecuteRecordsCount(string exportConfigFilePath, string schemaFilePath, IOrganizationService service, BackgroundWorker worker, DataGridView gridView)
{
worker.ThrowArgumentNullExceptionIfNull(nameof(worker));
gridView.ThrowArgumentNullExceptionIfNull(nameof(gridView));
service.ThrowArgumentNullExceptionIfNull(nameof(service));
CrmExporterConfig crmExporterConfig = CrmExporterConfig.GetConfiguration(exportConfigFilePath);
CrmSchemaConfiguration crmSchemaConfiguration = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
List <RecordCountModel> entityWrapperList = new List <RecordCountModel>();
foreach (var item in crmSchemaConfiguration.Entities)
{
string fetchXml = FetchXmlTemplate;
string filters = crmExporterConfig.CrmMigrationToolSchemaFilters != null && crmExporterConfig.CrmMigrationToolSchemaFilters.ContainsKey(item.Name) ? crmExporterConfig.CrmMigrationToolSchemaFilters[item.Name] : string.Empty;
fetchXml = !string.IsNullOrEmpty(filters) ? fetchXml.Replace("{filter}", filters) : fetchXml.Replace("{filter}", string.Empty);
fetchXml = fetchXml.Replace("{entity}", item.Name);
// Convert the FetchXML into a query expression.
var conversionRequest = new FetchXmlToQueryExpressionRequest
{
FetchXml = fetchXml
};
var conversionResponse =
(FetchXmlToQueryExpressionResponse)service.Execute(conversionRequest);
QueryExpression queryExpression = conversionResponse.Query;
queryExpression.ColumnSet = new ColumnSet(false);
worker.ReportProgress(0, $"Counting... {item.Name}");
var results = service.GetDataByQuery(queryExpression, 5000, false).TotalRecordCount;
worker.ReportProgress(0, $"{item.Name} record count: {results}");
entityWrapperList.Add(new RecordCountModel {
EntityName = item.Name, RecordCount = results
});
gridView.DataSource = null;
gridView.Refresh();
gridView.DataSource = entityWrapperList;
gridView.Columns[0].Width = 250;
gridView.Columns[1].Width = 250;
}
return(entityWrapperList);
}
public void SaveBatchDataToStoreTest()
{
if (Directory.Exists(extractPath))
{
Directory.Delete(extractPath, true);
}
Directory.CreateDirectory(extractPath);
CrmSchemaConfiguration schemaConfig = GetSchema();
DataFileStoreWriterCsv store = new DataFileStoreWriterCsv(new ConsoleLogger(), "test", extractPath, null, schemaConfig);
List <EntityWrapper> entities = PrepareEntities();
FluentActions.Invoking(() => store.SaveBatchDataToStore(entities))
.Should()
.NotThrow();
}
public void DataExportTest()
{
ConsoleLogger.LogLevel = 5;
var orgService = SourceService;
var repo = new EntityRepository(orgService, new ServiceRetryExecutor());
var config = GetExporterConfig();
string exportConfigPath = Path.Combine(SchemaFolderPathFull, $"{ConfigPrefix}ExportConfig.json");
if (File.Exists(exportConfigPath))
{
File.Delete(exportConfigPath);
}
config.SaveConfiguration(exportConfigPath);
if (Directory.Exists(ExtractedDataPathFull))
{
Directory.Delete(ExtractedDataPathFull, true);
}
Directory.CreateDirectory(ExtractedDataPathFull);
CrmFileDataExporter fileExporter = new CrmFileDataExporter(new ConsoleLogger(), repo, config, new CancellationToken(false));
FluentActions.Invoking(() => fileExporter.MigrateData())
.Should()
.NotThrow();
if (useCSV)
{
var schema = CrmSchemaConfiguration.ReadFromFile(SchemaPathFull);
CrmFileDataExporterCsv fileExporterCsv = new CrmFileDataExporterCsv(new ConsoleLogger(), repo, config, schema, new CancellationToken(false));
FluentActions.Invoking(() => fileExporterCsv.MigrateData())
.Should()
.NotThrow();
}
}
public void Setup()
{
InitializeProperties();
var entities = new List <CrmEntity>()
{
new CrmEntity(), new CrmEntity()
};
schemaConfig = new CrmSchemaConfiguration();
schemaConfig.Entities.AddRange(entities);
dataCrmStoreReader = new DataCrmStoreReader(MockLogger.Object, MockEntityRepo.Object, 500, 500, 1000, true, new List <string>(), EmptyFieldsToObfuscate);
string filePrefix = "Test";
string filesPath = "TestData";
List <string> excludedFields = new List <string>();
dataCrmStoreWriter = new DataFileStoreWriterCsv(MockLogger.Object, filePrefix, filesPath, excludedFields, schemaConfig);
}
private Tuple <List <Entity>, List <Entity> > GetFirstEntities(string filePrefix)
{
string extractedPath = Path.Combine(TestBase.GetWorkiongFolderPath(), "TestData");
string extractFolder = Path.Combine(extractedPath, "ExtractedData");
string schemaFilePath = Path.Combine(extractedPath, "usersettingsschema.xml");
CrmSchemaConfiguration schemaConfig = CrmSchemaConfiguration.ReadFromFile(schemaFilePath);
DataFileStoreReaderCsv store = new DataFileStoreReaderCsv(new ConsoleLogger(), filePrefix, extractFolder, schemaConfig);
var batch = store.ReadBatchDataFromStore();
List <Entity> firstEnt = batch.Select(p => p.OriginalEntity).ToList();
DataFileStoreReader storeJson = new DataFileStoreReader(new ConsoleLogger(), filePrefix, extractFolder);
var batchJson = storeJson.ReadBatchDataFromStore();
List <Entity> firstEntJson = batchJson.Select(p => p.OriginalEntity).ToList();
return(new Tuple <List <Entity>, List <Entity> >(firstEnt, firstEntJson));
}
public void ExportData(ExportSettings exportSettings)
{
if (exportSettings is null)
{
throw new ArgumentNullException(nameof(exportSettings));
}
var tokenSource = new CancellationTokenSource();
var repo = new EntityRepository(exportSettings.EnvironmentConnection, new ServiceRetryExecutor());
if (!string.IsNullOrEmpty(exportSettings.ExportConfigPath))
{
exportConfig = CrmExporterConfig.GetConfiguration(exportSettings.ExportConfigPath);
InjectAdditionalValuesIntoTheExportConfig(exportConfig, exportSettings);
}
else
{
exportConfig = new CrmExporterConfig
{
BatchSize = Convert.ToInt32(exportSettings.BatchSize),
PageSize = 5000,
TopCount = Convert.ToInt32(1000000),
OnlyActiveRecords = !exportSettings.ExportInactiveRecords,
JsonFolderPath = exportSettings.SavePath,
OneEntityPerBatch = true,
FilePrefix = "ExtractedData",
SeperateFilesPerEntity = true,
FetchXMLFolderPath = string.Empty
};
exportConfig.CrmMigrationToolSchemaPaths.Clear();
exportConfig.CrmMigrationToolSchemaPaths.Add(exportSettings.SchemaPath);
}
var schema = CrmSchemaConfiguration.ReadFromFile(exportSettings.SchemaPath);
var exporter = migratorFactory.GetCrmDataMigrator(exportSettings.DataFormat, logger, repo, exportConfig, tokenSource.Token, schema);
exporter.MigrateData();
}
public DataFileStoreWriterCsv(ILogger logger, string filePrefix, string filesPath, List <string> excludedFields, CrmSchemaConfiguration schemaConfig)
{
logger.ThrowArgumentNullExceptionIfNull(nameof(logger));
filePrefix.ThrowArgumentNullExceptionIfNull(nameof(filePrefix));
filesPath.ThrowArgumentNullExceptionIfNull(nameof(filesPath));
if (schemaConfig == null || !schemaConfig.Entities.Any())
{
throw new ArgumentNullException(nameof(schemaConfig));
}
this.excludedFields = excludedFields;
this.logger = logger;
this.filePrefix = filePrefix;
this.filesPath = filesPath;
seperateFilesPerEntity = true;
this.schemaConfig = schemaConfig;
if (this.excludedFields == null)
{
this.excludedFields = new List <string>();
}
}
public void StartSingleThreadedExport(CrmExporterConfig exportConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
var orgService = connectionHelper.GetOrganizationalService(connectionString);
logger.LogInfo("Connectd to instance " + connectionString);
var entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor());
if (useCsv)
{
var fileExporter = new CrmFileDataExporterCsv(logger, entityRepo, exportConfig, schemaConfig, cancellationToken.Token);
fileExporter.MigrateData();
}
else
{
var fileExporter = new CrmFileDataExporter(logger, entityRepo, exportConfig, cancellationToken.Token);
fileExporter.MigrateData();
}
}
public CrmFileDataExporterCsv(ILogger logger, IEntityRepository entityRepo, CrmExporterConfig exporterConfig, CrmSchemaConfiguration schemaConfig, CancellationToken token)
: base(
logger,
new DataCrmStoreReader(logger, entityRepo, exporterConfig),
new DataFileStoreWriterCsv(logger, exporterConfig, schemaConfig),
token)
{
}
protected override void ProcessRecord()
{
base.ProcessRecord();
var logger = new CmdletLogger(TreatWarningsAsErrors);
try
{
logger.LogInfo("About to start importing data from Dynamics365");
var manager = new Dynamics365DataManager();
var cancellationTokenSource = new CancellationTokenSource();
var importConfig = new CrmImportConfig();
if (!string.IsNullOrWhiteSpace(ConfigFilePath))
{
if (!File.Exists(ConfigFilePath))
{
WriteWarning($"Import config file path does not exist, will be ignored {ConfigFilePath}");
}
else
{
importConfig = CrmImportConfig.GetConfiguration(ConfigFilePath);
}
}
PopulateConfigFile(importConfig);
if (!Directory.Exists(JsonFolderPath))
{
WriteWarning($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
throw new DirectoryNotFoundException($"JsonFolderPath {JsonFolderPath} does not exist, cannot continue!");
}
CrmSchemaConfiguration schemaConfig = null;
if (CsvImport)
{
if (string.IsNullOrWhiteSpace(SchemaFilePath))
{
throw new ConfigurationException("Schema file is required for CSV Import!");
}
schemaConfig = CrmSchemaConfiguration.ReadFromFile(SchemaFilePath);
logger.LogInfo("Using Csv import");
}
else
{
logger.LogInfo("Using JSon import");
}
if (MaxThreads > 1)
{
var result = manager.StartImport(importConfig, logger, cancellationTokenSource, ConnectionString, MaxThreads, CsvImport, schemaConfig)
.ContinueWith(a =>
{
logger.LogInfo("Dynamics365 data import completed successfully.");
},
cancellationTokenSource.Token);
result.Wait(cancellationTokenSource.Token);
}
else
{
manager.StartSingleThreadedImport(importConfig, new CmdletLoggerPS(this, TreatWarningsAsErrors), cancellationTokenSource, ConnectionString, CsvImport, schemaConfig);
}
}
catch (Exception exception)
{
var errorMessage = $"Dynamics365 data import failed: {exception.Message}";
logger.LogVerbose(errorMessage);
logger.LogError(errorMessage);
throw;
}
}
public void StartSingleThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
var orgService = connectionHelper.GetOrganizationalService(connectionString);
logger.Info("Starting Single Threaded processing, you must configure connection string for multithreaded processing adn set up max threads to more than 1");
var entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor());
CrmGenericImporter fileImporter = null;
if (!useCsv)
{
fileImporter = new CrmFileDataImporter(logger, entityRepo, importConfig, cancellationToken.Token);
}
else
{
fileImporter = new CrmFileDataImporterCsv(logger, entityRepo, importConfig, schemaConfig, cancellationToken.Token);
}
fileImporter.MigrateData();
}
private static void StartMultiThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
logger.Info($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
var cnt = Convert.ToInt32(maxThreads);
while (cnt > 0)
{
cnt--;
repos.Add(new EntityRepository(connectionHelper.GetOrganizationalService(connectionString), new ServiceRetryExecutor()));
logger.Info("New connection created to " + connectionString);
}
CrmGenericImporter fileImporter = null;
if (!useCsv)
{
fileImporter = new CrmFileDataImporter(logger, repos, importConfig, cancellationToken.Token);
}
else
{
fileImporter = new CrmFileDataImporterCsv(logger, repos, importConfig, schemaConfig, cancellationToken.Token);
}
fileImporter.MigrateData();
}
public async Task StartImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var importTask = Task.Run(() =>
{
logger.Info("Connectd to instance " + connectionString);
if (maxThreads > 1 && !string.IsNullOrWhiteSpace(connectionString))
{
StartMultiThreadedImport(importConfig, logger, cancellationToken, connectionString, maxThreads, useCsv, schemaConfig);
}
else
{
StartSingleThreadedImport(importConfig, logger, cancellationToken, connectionString, useCsv, schemaConfig);
}
});
await importTask;
}
private static void ImportData(string connectionString, string schemaPath)
{
Console.WriteLine("Import Started");
using (var tokenSource = new CancellationTokenSource())
{
using (var serviceClient = new CrmServiceClient(connectionString))
{
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor());
int threadCount = Settings.Default.ThreadCount;
var logger = new ConsoleLogger();
if (threadCount > 1)
{
List <IEntityRepository> entRepos = new List <IEntityRepository>()
{
entityRepo
};
while (threadCount > 1)
{
threadCount--;
var newServiceClient = serviceClient.Clone();
if (newServiceClient != null)
{
entRepos.Add(new EntityRepository(serviceClient.Clone(), new ServiceRetryExecutor()));
}
else
{
Console.WriteLine("CrmServiceClient Clone() operation is only supported for OAUTH connections, single thread will be used as clone returned null");
}
}
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entRepos, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
// Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entRepos, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
}
else
{
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entityRepo, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
// Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entityRepo, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
}
}
}
Console.WriteLine("Import Finished");
}
