public static void ImportData(string connectionString, string schemaPath, string exportFolderPath)
{
Console.WriteLine("Import Started");
var tokenSource = new CancellationTokenSource();
var serviceClient = new CrmServiceClient(connectionString);
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor());
var logger = new ConsoleLogger();
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entityRepo, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
//Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entityRepo, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
Console.WriteLine("Import Finished");
}
private static void StartMultiThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, int maxThreads, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
logger.Info($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
var cnt = Convert.ToInt32(maxThreads);
while (cnt > 0)
{
cnt--;
repos.Add(new EntityRepository(connectionHelper.GetOrganizationalService(connectionString), new ServiceRetryExecutor()));
logger.Info("New connection created to " + connectionString);
}
CrmGenericImporter fileImporter = null;
if (!useCsv)
{
fileImporter = new CrmFileDataImporter(logger, repos, importConfig, cancellationToken.Token);
}
else
{
fileImporter = new CrmFileDataImporterCsv(logger, repos, importConfig, schemaConfig, cancellationToken.Token);
}
fileImporter.MigrateData();
}
private string DataImport(bool useCsv)
{
ConsoleLogger.LogLevel = 5;
var logger = new ConsoleLogger();
List <IEntityRepository> entityRepos = new List <IEntityRepository>();
int cnt = 0;
while (entityRepos.Count < threadsNo)
{
if (useFakeRepo)
{
entityRepos.Add(new EntityRepositoryFake(targetServices[cnt], new ServiceRetryExecutor()));
}
else
{
entityRepos.Add(new EntityRepository(targetServices[cnt], new ServiceRetryExecutor()));
}
cnt++;
}
var importConfig = GetImporterConfig();
string importConfigPath = Path.Combine(SchemaFolderPathFull, ConfigPrefix + "ImportConfig.json");
if (File.Exists(importConfigPath))
{
File.Delete(importConfigPath);
}
importConfig.SaveConfiguration(importConfigPath);
CrmGenericImporter fileImporter;
if (useCsv)
{
var schema = CrmSchemaConfiguration.ReadFromFile(SchemaPathFull);
fileImporter = new CrmFileDataImporterCsv(logger, entityRepos, importConfig, schema, new CancellationToken(false));
}
else
{
fileImporter = new CrmFileDataImporter(logger, entityRepos, importConfig, new CancellationToken(false));
}
var watch = Stopwatch.StartNew();
fileImporter.MigrateData();
watch.Stop();
var message = $"Importing Data completed, took {watch.Elapsed.TotalSeconds} seconds";
Debug.WriteLine(message);
return(message);
}
public void PerformImportAction(string importSchemaFilePath, int maxThreads, bool jsonFormat, Capgemini.DataMigration.Core.ILogger currentLogger, IEntityRepositoryService entityRepositoryService, CrmImportConfig currentImportConfig, CancellationTokenSource tokenSource)
{
try
{
if (maxThreads > 1)
{
currentLogger.LogInfo($"Starting MultiThreaded Processing, using {maxThreads} threads");
var repos = new List <IEntityRepository>();
int threadCount = maxThreads;
while (threadCount > 0)
{
threadCount--;
repos.Add(entityRepositoryService.InstantiateEntityRepository(true));
}
var fileExporter = new CrmFileDataImporter(currentLogger, repos, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
currentLogger.LogInfo("Starting Single Threaded processing, you must set up max threads to more than 1");
var entityRepo = entityRepositoryService.InstantiateEntityRepository(false);
if (jsonFormat)
{
var fileExporter = new CrmFileDataImporter(currentLogger, entityRepo, currentImportConfig, tokenSource.Token);
fileExporter.MigrateData();
}
else
{
var schema = CrmSchemaConfiguration.ReadFromFile(importSchemaFilePath);
var fileExporter = new CrmFileDataImporterCsv(currentLogger, entityRepo, currentImportConfig, schema, tokenSource.Token);
fileExporter.MigrateData();
}
}
}
catch (Exception ex)
{
currentLogger.LogError($"Critical import error, processing stopped: {ex.Message}");
throw;
}
}
public void StartSingleThreadedImport(CrmImportConfig importConfig, ILogger logger, CancellationTokenSource cancellationToken, string connectionString, bool useCsv, CrmSchemaConfiguration schemaConfig)
{
var connectionHelper = new ConnectionHelper();
var orgService = connectionHelper.GetOrganizationalService(connectionString);
logger.Info("Starting Single Threaded processing, you must configure connection string for multithreaded processing adn set up max threads to more than 1");
var entityRepo = new EntityRepository(orgService, new ServiceRetryExecutor());
CrmGenericImporter fileImporter = null;
if (!useCsv)
{
fileImporter = new CrmFileDataImporter(logger, entityRepo, importConfig, cancellationToken.Token);
}
else
{
fileImporter = new CrmFileDataImporterCsv(logger, entityRepo, importConfig, schemaConfig, cancellationToken.Token);
}
fileImporter.MigrateData();
}
private static void ImportData(string connectionString, string schemaPath)
{
Console.WriteLine("Import Started");
using (var tokenSource = new CancellationTokenSource())
{
using (var serviceClient = new CrmServiceClient(connectionString))
{
var entityRepo = new EntityRepository(serviceClient, new ServiceRetryExecutor());
int threadCount = Settings.Default.ThreadCount;
var logger = new ConsoleLogger();
if (threadCount > 1)
{
List <IEntityRepository> entRepos = new List <IEntityRepository>()
{
entityRepo
};
while (threadCount > 1)
{
threadCount--;
var newServiceClient = serviceClient.Clone();
if (newServiceClient != null)
{
entRepos.Add(new EntityRepository(serviceClient.Clone(), new ServiceRetryExecutor()));
}
else
{
Console.WriteLine("CrmServiceClient Clone() operation is only supported for OAUTH connections, single thread will be used as clone returned null");
}
}
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entRepos, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
// Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entRepos, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
}
else
{
if (!Settings.Default.UseCsvImport)
{
// Json Import
var fileImporterJson = new CrmFileDataImporter(logger, entityRepo, GetImportConfig(), tokenSource.Token);
fileImporterJson.MigrateData();
}
else
{
// Csv Import
var schema = CrmSchemaConfiguration.ReadFromFile(schemaPath);
var fileImporterCsv = new CrmFileDataImporterCsv(logger, entityRepo, GetImportConfig(), schema, tokenSource.Token);
fileImporterCsv.MigrateData();
}
}
}
}
Console.WriteLine("Import Finished");
}
