public MainFormPresenter(
IMainFormView view,
IDialogService dialogService,
JsonFilesImporter jsonFilesImporter,
SqlImporter sqlImporter)
{
_dialogService = dialogService;
_jsonFilesImporter = jsonFilesImporter;
_sqlImporter = sqlImporter;
View = view;
View.OpenJsonFolderClick += OpenJsonFolder;
View.AddFilesAsAccountsInfoList += AddFilesAsAccountsInfoList;
View.AddFilesAsTanksInfoList += AddFilesAsTanksInfoList;
View.StartJsonConvert += StartJsonConvert;
View.FetchSqlAccounts += FetchSqlAccounts;
View.StartSqlImport += StartSqlImport;
}
public void SetUp()
{
_cosmosFeed = new Mock <ICosmosDbFeedIterator <PublishedProvider> >();
_importContext = new Mock <ISqlImportContext>();
_importContextBuilder = new Mock <ISqlImportContextBuilder>();
_dataTableImporter = new Mock <IDataTableImporter>();
_fundingLineOneProfilingDataTableBuilder = NewDataTableBuilder();
_fundingLineTwoProfilingDataTableBuilder = NewDataTableBuilder();
_paymentFundingLineDataTableBuilder = NewDataTableBuilder();
_informationFundingLineDataTableBuilder = NewDataTableBuilder();
_calculationDataTableBuilder = NewDataTableBuilder();
_publishedProviderVersionDataTableBuilder = NewDataTableBuilder();
_fundingDataTableBuilder = NewDataTableBuilder();
_importContext.Setup(_ => _.Providers)
.Returns(_publishedProviderVersionDataTableBuilder.Object);
_importContext.Setup(_ => _.Calculations)
.Returns(_calculationDataTableBuilder.Object);
_importContext.Setup(_ => _.Funding)
.Returns(_fundingDataTableBuilder.Object);
_importContext.Setup(_ => _.Profiling)
.Returns(new Dictionary <uint, IDataTableBuilder <PublishedProviderVersion> >
{
{
(uint)NewRandomNumber(), _fundingLineOneProfilingDataTableBuilder.Object
},
{
(uint)NewRandomNumber(), _fundingLineTwoProfilingDataTableBuilder.Object
},
});
_importContext.Setup(_ => _.PaymentFundingLines)
.Returns(_paymentFundingLineDataTableBuilder.Object);
_importContext.Setup(_ => _.InformationFundingLines)
.Returns(_informationFundingLineDataTableBuilder.Object);
_importContext.Setup(_ => _.Documents)
.Returns(_cosmosFeed.Object);
_sqlImporter = new SqlImporter(new ProducerConsumerFactory(),
_importContextBuilder.Object,
_dataTableImporter.Object,
Logger.None);
}
public async Task Run(string[] args)
{
var importers = LoadImporters(Configuration);
string dataDirectory;
if (args.Length != 0)
{
dataDirectory = args[0];
}
else
{
dataDirectory = Configuration["dataDirectory"];
}
foreach (var importer in importers.Where(i => i.IsActive))
{
if (importer.Type == ImporterType.TextImporter && importer.ImportModel == "TaxonomicUnit")
{
var processor = _importTransformOrchestrator.GetPlantInfoProcessor;
var transformer = new USDATransformer();
var startRow = 1;
var batchSize = 100;
await processor.InitializeOrigin(transformer.Origin);
await processor.InitializeLifeforms();
await processor.InitializeTaxons();
var dataFile = FileHelpers.GetDatafileName(importer.Filename, dataDirectory);
var textImporter = new TextImporter <Checklist>(dataFile, importer.HasHeaders);
var row = 1;
var checklists = new List <Checklist>();
await foreach (var result in textImporter.Import())
{
row++;
if (row < startRow)
{
continue;
}
else if (row % batchSize != 0)
{
checklists.Add(result);
}
else
{
if (checklists.Any())
{
await ProcessChecklists(transformer, processor, checklists);
checklists.Clear();
}
}
if (checklists.Any())
{
await ProcessChecklists(transformer, processor, checklists);
checklists.Clear();
}
}
}
else if (importer.Type == ImporterType.SqlImporter && importer.ImportModel == "TaxonomicUnit")
{
var processor = _importTransformOrchestrator.GetPlantInfoProcessor;
var sqlImporter = new SqlImporter <TaxonomicUnit>(importer.ConnectionString, importer.SqlQuery);
var transformer = new ITISPlantInfoTransformer();
var startRow = 0;
var batchSize = 100;
var row = 0;
var taxonomicUnits = new List <TaxonomicUnit>();
await processor.InitializeOrigin(transformer.Origin);
await processor.InitializeLifeforms();
await processor.InitializeTaxons();
await foreach (var result in sqlImporter.Import())
{
row++;
if (row < startRow)
{
continue;
}
else if (row % batchSize != 0)
{
taxonomicUnits.Add(result);
}
else
{
if (taxonomicUnits.Any())
{
await ProcessTaxonomicUnits(transformer, processor, taxonomicUnits);
taxonomicUnits.Clear();
}
}
}
if (taxonomicUnits.Any())
{
await ProcessTaxonomicUnits(transformer, processor, taxonomicUnits);
taxonomicUnits.Clear();
}
}
else if (importer.Type == ImporterType.SqlImporter && importer.ImportModel == "Vernacular")
{
var processor = _importTransformOrchestrator.GetSynonymProcessor;
var sqlImporter = new SqlImporter <Vernacular>(importer.ConnectionString, importer.SqlQuery);
var transformer = new ITISSynonymTransformer();
var startRow = 0;
var batchSize = 500;
var row = 0;
var vernaculars = new List <Vernacular>();
await processor.InitializeOrigin(transformer.Origin);
await processor.InitializeTaxons();
await foreach (var result in sqlImporter.Import())
{
row++;
if (row < startRow)
{
#pragma warning disable S3626 // Jump statements should not be redundant
continue;
#pragma warning restore S3626 // Jump statements should not be redundant
}
else if (row % batchSize != 0)
{
vernaculars.Add(result);
}
else
{
await ProcessSynonyms(transformer, processor, vernaculars);
vernaculars.Clear();
}
}
if (vernaculars.Any())
{
await ProcessSynonyms(transformer, processor, vernaculars);
vernaculars.Clear();
}
}
else if (importer.Type == ImporterType.EFImporter && importer.ImportModel == "PlantInfo")
{
var processor = _importTransformOrchestrator.GetElasticPlantInfoProcessor;
var finished = false;
var counter = 0;
var counterEnd = 40000;
while (!finished && counter < counterEnd)
{
//var response = await processor.Process(counter, counter + 1000);
var response = await processor.ProcessSome(counter, 1000);
Console.WriteLine($"Successes: {response.Successes} / Failures: {response.Failures}");
if (response.Failures > 0)
{
throw new Exception("Errors! Noooooo!");
}
else if (response.Successes == 0)
{
finished = true;
}
counter += 1000;
}
}
else if (importer.Type == ImporterType.EFImporter && importer.ImportModel == "Specimen")
{
var processor = _importTransformOrchestrator.GetElasticSpecimenProcessor;
var finished = false;
var counter = 0;
var counterEnd = 1000000;
while (!finished && counter < counterEnd)
{
var response = await processor.Process(counter, counter + 1000);
Console.WriteLine($"Successes: {response.Successes} / Failures: {response.Failures}");
if (response.Failures > 0)
{
throw new Exception("Errors! Noooooo!");
}
else if (response.Successes == 0)
{
finished = true;
}
counter += 1000;
}
}
else if (importer.Type == ImporterType.JsonImporter && importer.ImportModel == "PlantsFile")
{
var transformer = new NatureServeTransformer();
var dataFile = FileHelpers.GetDatafileName(importer.Filename, dataDirectory);
var textImporter = new JsonImporter <IEnumerable <Plant> >(dataFile);
var plants = textImporter.ImportObjectAsync();
var plantInfos = new List <PlantInfo>();
List <Taxon> taxons;
List <Lifeform> lifeforms;
List <Origin> origins;
foreach (var plant in plants)
{
plantInfos.Add(transformer.Transform(plant));
}
var processor = _importTransformOrchestrator.GetNatureServePlantInfoProcessor();
await processor.InitializeOrigin(transformer.Origin);
await processor.InitializeLifeforms();
await processor.InitializeTaxons();
await processor.InitializeOrigins();
taxons = processor.Taxons;
lifeforms = processor.Lifeforms;
origins = processor.Origins;
var batchSize = 500;
var finished = false;
var start = 0;
var counter = start;
var counterEnd = 40000;
plantInfos = plantInfos.OrderBy(p => p.Origin.ExternalId).ToList();
while (!finished && counter < counterEnd)
{
if (counter > start)
{
processor = _importTransformOrchestrator.GetNatureServePlantInfoProcessor(lifeforms, taxons, origins);
await processor.InitializeOrigin(transformer.Origin);
}
var batch = plantInfos.Skip(counter).Take(batchSize);
if (!batch.Any())
{
finished = true;
continue;
}
var result = await processor.Process(batch);
Console.WriteLine($"Processed: {result.Count()} plantInfos - ended on {batch.Last().ScientificName}");
counter += batchSize;
}
}
}
}
