public async Task <ImportReport> PerformImport(ExportType exportType, bool fetchLiveData, APICredentials credentials, CoreReferenceData coreRefData, string outputPath, IImportProvider provider, bool cacheInputData, bool fetchExistingFromAPI = false) { var p = ((BaseImportProvider)provider); p.ExportType = exportType; ImportReport resultReport = new ImportReport(); resultReport.ProviderDetails = p; try { bool loadOK = false; if (p.ImportInitialisationRequired && p is IImportProviderWithInit) { ((IImportProviderWithInit)provider).InitImportProvider(); } if (fetchLiveData && p.IsAutoRefreshed && !String.IsNullOrEmpty(p.AutoRefreshURL)) { Log("Loading input data from URL.."); loadOK = p.LoadInputFromURL(p.AutoRefreshURL); } else { if (p.IsStringData && !p.UseCustomReader) { Log("Loading input data from file.."); loadOK = p.LoadInputFromFile(p.InputPath); } else { //binary streams pass as OK by default loadOK = true; } } if (!loadOK) { //failed to load Log("Failed to load input data."); throw new Exception("Failed to fetch input data"); } else { if (fetchLiveData && cacheInputData) { //save input data p.SaveInputFile(p.InputPath); } } List <ChargePoint> duplicatesList = new List <ChargePoint>(); Log("Processing input.."); var list = provider.Process(coreRefData); int numAdded = 0; int numUpdated = 0; if (list.Count > 0) { if (p.MergeDuplicatePOIEquipment) { Log("Merging Equipment from Duplicate POIs"); list = MergeDuplicatePOIEquipment(list); } if (!p.IncludeInvalidPOIs) { Log("Cleaning invalid POIs"); var invalidPOIs = new List <ChargePoint>(); foreach (var poi in list) { if (!BaseImportProvider.IsPOIValidForImport(poi)) { invalidPOIs.Add(poi); } } foreach (var poi in invalidPOIs) { list.Remove(poi); } } GC.Collect(); List <ChargePoint> finalList = new List <ChargePoint>(); if (!p.SkipDeduplication) { Log("De-Deuplicating list (" + p.ProviderName + ":: " + list.Count + " Items).."); //de-duplicate and clean list based on existing data //TODO: take original and replace in final update list, setting relevant updated properties (merge) and status finalList = await DeDuplicateList(list.ToList(), true, coreRefData, resultReport, p.AllowDuplicatePOIWithDifferentOperator, fetchExistingFromAPI); //var finalList = list; } else { //skip deduplication finalList = list.ToList(); } if (ImportUpdatesOnly) { finalList = finalList.Where(l => l.ID > 0).ToList(); } //finalList = client.GetLocations(new SearchFilters { MaxResults = 10000 }); GC.Collect(); //export/apply updates if (p.ExportType == ExportType.XML) { Log("Exporting XML.."); //output xml p.ExportXMLFile(finalList, outputPath + p.OutputNamePrefix + ".xml"); } if (p.ExportType == ExportType.CSV) { Log("Exporting CSV.."); //output csv p.ExportCSVFile(finalList, outputPath + p.OutputNamePrefix + ".csv"); } if (p.ExportType == ExportType.JSON) { Log("Exporting JSON.."); //output json p.ExportJSONFile(finalList, outputPath + p.OutputNamePrefix + ".json"); } if (p.ExportType == ExportType.API && p.IsProductionReady) { //publish list of locations to OCM via API OCMClient ocmClient = new OCMClient(IsSandboxedAPIMode); Log("Publishing via API.."); foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.Country != null)) { ocmClient.UpdateItem(cp, credentials); if (cp.ID == 0) { numAdded++; } else { numUpdated++; } } } if (p.ExportType == ExportType.POIModelList) { //result report contains POI lists } } Log("Import Processed:" + provider.GetProviderName() + " Added:" + numAdded + " Updated:" + numUpdated); } catch (Exception exp) { Log("Import Failed:" + provider.GetProviderName() + " ::" + exp.ToString()); } resultReport.Log = ""; resultReport.Log += p.ProcessingLog; resultReport.Log += ImportLog; return(resultReport); }
public async Task <ImportReport> PerformImport(ExportType exportType, bool fetchLiveData, APICredentials credentials, CoreReferenceData coreRefData, string outputPath, IImportProvider provider) { var p = ((BaseImportProvider)provider); p.ExportType = exportType; ImportReport resultReport = new ImportReport(); resultReport.ProviderDetails = p; try { bool loadOK = false; if (fetchLiveData && p.IsAutoRefreshed && !String.IsNullOrEmpty(p.AutoRefreshURL)) { p.Log("Loading input data from URL.."); loadOK = p.LoadInputFromURL(p.AutoRefreshURL); } else { if (p.IsStringData) { p.Log("Loading input data from file.."); loadOK = p.LoadInputFromFile(p.InputPath); } else { //binary streams pass as OK by default loadOK = true; } } if (!loadOK) { //failed to load p.Log("Failed to load input data."); throw new Exception("Failed to fetch input data"); } List <ChargePoint> duplicatesList = new List <ChargePoint>(); p.Log("Processing input.."); var list = provider.Process(coreRefData); int numAdded = 0; int numUpdated = 0; if (list.Count > 0) { p.Log("De-Deuplicating list (" + p.ProviderName + ":: " + list.Count + " Items).."); //de-duplicate and clean list based on existing data //TODO: take original and replace in final update list, setting relevant updated properties (merge) and status var finalList = await DeDuplicateList(list, true, coreRefData, resultReport); //var finalList = list; if (ImportUpdatesOnly) { finalList = finalList.Where(l => l.ID > 0).ToList(); } //finalList = client.GetLocations(new SearchFilters { MaxResults = 10000 }); //export/apply updates if (p.ExportType == ExportType.XML) { p.Log("Exporting XML.."); //output xml p.ExportXMLFile(finalList, outputPath + p.OutputNamePrefix + ".xml"); } if (p.ExportType == ExportType.CSV) { p.Log("Exporting CSV.."); //output csv p.ExportCSVFile(finalList, outputPath + p.OutputNamePrefix + ".csv"); } if (p.ExportType == ExportType.JSON) { p.Log("Exporting JSON.."); //output json p.ExportJSONFile(finalList, outputPath + p.OutputNamePrefix + ".json"); } if (p.ExportType == ExportType.API && p.IsProductionReady) { //publish list of locations to OCM via API OCMClient ocmClient = new OCMClient(IsSandboxedAPIMode); p.Log("Publishing via API.."); foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.Country != null)) { ocmClient.UpdateItem(cp, credentials); if (cp.ID == 0) { numAdded++; } else { numUpdated++; } } } if (p.ExportType == ExportType.POIModelList) { resultReport.Added = finalList.Where(cp => cp.ID == 0).ToList(); resultReport.Updated = finalList.Where(cp => cp.ID > 0).ToList(); } } p.Log("Import Processed:" + provider.GetProviderName() + " Added:" + numAdded + " Updated:" + numUpdated); } catch (Exception exp) { p.Log("Import Failed:" + provider.GetProviderName() + " ::" + exp.ToString()); } return(resultReport); }
public async Task<ImportReport> PerformImport(ExportType exportType, bool fetchLiveData, APICredentials credentials, CoreReferenceData coreRefData, string outputPath, IImportProvider provider, bool cacheInputData) { var p = ((BaseImportProvider)provider); p.ExportType = exportType; ImportReport resultReport = new ImportReport(); resultReport.ProviderDetails = p; try { bool loadOK = false; if (p.ImportInitialisationRequired && p is IImportProviderWithInit) { ((IImportProviderWithInit)provider).InitImportProvider(); } if (fetchLiveData && p.IsAutoRefreshed && !String.IsNullOrEmpty(p.AutoRefreshURL)) { Log("Loading input data from URL.."); loadOK = p.LoadInputFromURL(p.AutoRefreshURL); } else { if (p.IsStringData && !p.UseCustomReader) { Log("Loading input data from file.."); loadOK = p.LoadInputFromFile(p.InputPath); } else { //binary streams pass as OK by default loadOK = true; } } if (!loadOK) { //failed to load Log("Failed to load input data."); throw new Exception("Failed to fetch input data"); } else { if (fetchLiveData && cacheInputData) { //save input data p.SaveInputFile(p.InputPath); } } List<ChargePoint> duplicatesList = new List<ChargePoint>(); Log("Processing input.."); var list = provider.Process(coreRefData); int numAdded = 0; int numUpdated = 0; if (list.Count > 0) { if (p.MergeDuplicatePOIEquipment) { Log("Merging Equipment from Duplicate POIs"); list = MergeDuplicatePOIEquipment(list); } if (!p.IncludeInvalidPOIs) { Log("Cleaning invalid POIs"); var invalidPOIs = new List<ChargePoint>(); foreach (var poi in list) { if (!BaseImportProvider.IsPOIValidForImport(poi)) { invalidPOIs.Add(poi); } } foreach (var poi in invalidPOIs) { list.Remove(poi); } } List<ChargePoint> finalList = new List<ChargePoint>(); if (!p.SkipDeduplication) { Log("De-Deuplicating list (" + p.ProviderName + ":: " + list.Count + " Items).."); //de-duplicate and clean list based on existing data //TODO: take original and replace in final update list, setting relevant updated properties (merge) and status finalList = await DeDuplicateList(list.ToList(), true, coreRefData, resultReport, p.AllowDuplicatePOIWithDifferentOperator); //var finalList = list; } else { //skip deduplication finalList = list.ToList(); } if (ImportUpdatesOnly) { finalList = finalList.Where(l => l.ID > 0).ToList(); } //finalList = client.GetLocations(new SearchFilters { MaxResults = 10000 }); //export/apply updates if (p.ExportType == ExportType.XML) { Log("Exporting XML.."); //output xml p.ExportXMLFile(finalList, outputPath + p.OutputNamePrefix + ".xml"); } if (p.ExportType == ExportType.CSV) { Log("Exporting CSV.."); //output csv p.ExportCSVFile(finalList, outputPath + p.OutputNamePrefix + ".csv"); } if (p.ExportType == ExportType.JSON) { Log("Exporting JSON.."); //output json p.ExportJSONFile(finalList, outputPath + p.OutputNamePrefix + ".json"); } if (p.ExportType == ExportType.API && p.IsProductionReady) { //publish list of locations to OCM via API OCMClient ocmClient = new OCMClient(IsSandboxedAPIMode); Log("Publishing via API.."); foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.Country != null)) { ocmClient.UpdateItem(cp, credentials); if (cp.ID == 0) { numAdded++; } else { numUpdated++; } } } if (p.ExportType == ExportType.POIModelList) { //result report contains POI lists } } Log("Import Processed:" + provider.GetProviderName() + " Added:" + numAdded + " Updated:" + numUpdated); } catch (Exception exp) { Log("Import Failed:" + provider.GetProviderName() + " ::" + exp.ToString()); } resultReport.Log = ""; resultReport.Log += p.ProcessingLog; resultReport.Log += ImportLog; return resultReport; }