public async Task <bool> PerformImportProcessing(ExportType exportType, string defaultDataPath, string apiIdentifier, string apiSessionToken, bool fetchLiveData)
{
OCMClient client = new OCMClient(IsSandboxedAPIMode);
var credentials = GetAPISessionCredentials(apiIdentifier, apiSessionToken);
CoreReferenceData coreRefData = null;
coreRefData = await client.GetCoreReferenceData();
string outputPath = "Data\\";
List <IImportProvider> providers = new List <IImportProvider>();
string inputDataPathPrefix = defaultDataPath;
// providers.Add(new ImportProvider_RWEMobility() { InputPath = inputDataPathPrefix + "rwe-mobility\\data.json.txt" });
//Working - Auto refreshed
//providers.Add(new ImportProvider_UKChargePointRegistry() { InputPath = inputDataPathPrefix + "chargepoints.dft.gov.uk\\data.json" });
/* providers.Add(new ImportProvider_BlinkNetwork() { InputPath = inputDataPathPrefix + "blinknetwork.com\\jsondata2.txt" });
* providers.Add(new ImportProvider_CarStations() { InputPath = inputDataPathPrefix + "carstations.com\\jsonresults.txt" });
* providers.Add(new ImportProvider_RWEMobility() { InputPath = inputDataPathPrefix + "rwe-mobility\\data.json.txt" });
*
* providers.Add(new ImportProvider_Mobie() { InputPath = inputDataPathPrefix + "mobie.pt\\data.json.txt" });
* providers.Add(new ImportProvider_AFDC() { InputPath = inputDataPathPrefix + "afdc\\data.json" });
* providers.Add(new ImportProvider_ESB_eCars() { InputPath = inputDataPathPrefix + "esb_ecars\\data.kml" });
*/
//providers.Add(new ImportProvider_AddEnergie(ImportProvider_AddEnergie.NetworkType.LeCircuitElectrique) { InputPath = inputDataPathPrefix + "addenergie\\StationsList.lecircuitelectrique.json" });
//providers.Add(new ImportProvider_AddEnergie(ImportProvider_AddEnergie.NetworkType.ReseauVER) { InputPath = inputDataPathPrefix + "addenergie\\StationsList.reseauver.json" });
//providers.Add(new ImportProvider_GenericExcel() { InputPath = inputDataPathPrefix + "catalonia\\catalonia.xlsx" , DefaultDataProvider=coreRefData.DataProviders.First(d=>d.ID==25)});
//Working -manual refresh
//providers.Add(new ImportProvider_NobilDotNo() { InputPath = inputDataPathPrefix + "nobil\\nobil.json.txt" });
//Dev
//providers.Add(new ImportProvider_CoulombChargepoint() { InputPath = inputDataPathPrefix + "coulomb\\data.json.txt" });
//providers.Add(new ImportProvider_ChademoGroup() { InputPath = inputDataPathPrefix + "chademo\\chademo_jp.kml", ImportType= ChademoImportType.Japan, DefaultCountryID=114});
//obsolete
//providers.Add(new ImportProvider_PODPoint() { InputPath = inputDataPathPrefix + "pod-point.com\\export.htm" });
//providers.Add(new ImportProvider_ChargeYourCar() { InputPath = inputDataPathPrefix + "chargeyourcar.org.uk\\data.htm" });
foreach (var provider in providers)
{
await PerformImport(exportType, fetchLiveData, credentials, coreRefData, outputPath, provider);
}
return(true);
}
public void GeocodingTest()
{
OCMClient client = new OCMClient(IsSandboxedAPIMode);
//get a few OCM listings
SearchFilters filters = new SearchFilters {
SubmissionStatusTypeIDs = new int[] { (int)StandardSubmissionStatusTypes.Submitted_Published }, CountryIDs = new int[] { 1 }, DataProviderIDs = new int[] { 1 }, MaxResults = 2000, EnableCaching = false
};
var poiList = client.GetLocations(filters);
/*
* GeocodingService g = new GeocodingService();
* List<GeolocationResult> list = new List<GeolocationResult>();
*
* //attempt OSM geocoding
* foreach (var poi in poiList)
* {
* try
* {
* System.Diagnostics.Debug.WriteLine("OCM-" + poi.ID + " : [" + poi.AddressInfo.Title + "] " + poi.AddressInfo.ToString());
*
* System.Diagnostics.Debug.WriteLine("OCM : LL: " + poi.AddressInfo.Latitude + "," + poi.AddressInfo.Longitude);
*
* var osm = g.GeolocateAddressInfo_OSM(poi.AddressInfo);
* System.Diagnostics.Debug.WriteLine("OSM : LL: " + osm.Latitude + "," + osm.Longitude);
* list.Add(osm);
*
* var mpq = g.GeolocateAddressInfo_MapquestOSM(poi.AddressInfo);
* System.Diagnostics.Debug.WriteLine("MPQ : LL: " + mpq.Latitude + "," + mpq.Longitude);
* list.Add(mpq);
* }
* catch (Exception exp)
* {
* System.Diagnostics.Debug.WriteLine("Exception during geocoding:" + exp.ToString());
* }
* System.Threading.Thread.Sleep(1000);
* }
*
* string json = JsonConvert.SerializeObject(list, Formatting.Indented);
* System.IO.File.WriteAllText("C:\\temp\\GeocodingResult.json", json);
* */
}
public async Task <bool> PerformImportProcessing(ExportType exportType, string defaultDataPath, string apiIdentifier, string apiSessionToken, bool fetchLiveData)
{
OCMClient client = new OCMClient(IsSandboxedAPIMode);
var credentials = GetAPISessionCredentials(apiIdentifier, apiSessionToken);
CoreReferenceData coreRefData = null;
coreRefData = await client.GetCoreReferenceData();
string outputPath = "Data\\";
List <IImportProvider> providers = new List <IImportProvider>();
string inputDataPathPrefix = defaultDataPath;
foreach (var provider in providers)
{
await PerformImport(exportType, fetchLiveData, credentials, coreRefData, outputPath, provider, false);
}
return(true);
}
static void Main(string[] args)
{
bool isAutomaticMode = true;
bool isAPIImportMode = false;
string importFolder = "";
string OCM_API_Identifier = null;
string OCM_API_SessionToken = null;
LogEvent("Starting Import:", true);
if (args.Length > 0)
{
LogEvent("Arguments supplied: ");
for (int i = 0; i < args.Length; i++)
{
string arg = args[i];
LogEvent("arg: " + arg);
if (arg == "-auto")
{
isAutomaticMode = true;
}
if (arg == "-api")
{
isAPIImportMode = true;
}
//if (arg == "-log") EnableLogging = true;
try
{
if (arg == "-api_identifier")
{
OCM_API_Identifier = args[i + 1];
}
if (arg == "-api_sessiontoken")
{
OCM_API_SessionToken = args[i + 1];
}
}
catch (Exception) { LogEvent("Invalid parameter supplied."); }
}
}
else
{
LogEvent("No Arguments supplied.");
}
if (isAutomaticMode)
{
bool actionPerformed = false;
ActionType mode = ActionType.ClientTest;
if (mode == ActionType.DataImport && isAPIImportMode == true && OCM_API_Identifier != null && OCM_API_SessionToken != null)
{
ExportType exportType = ExportType.API;
ImportManager importManager = new ImportManager(importFolder);
LogEvent("Performing Import, Publishing via API (" + OCM_API_Identifier + ":" + OCM_API_SessionToken + "): " + DateTime.UtcNow.ToShortTimeString());
Task <bool> processing = importManager.PerformImportProcessing(exportType, importFolder, OCM_API_Identifier, OCM_API_SessionToken, true);
processing.Wait();
LogEvent("Import Processed. Exiting. " + DateTime.UtcNow.ToShortTimeString());
actionPerformed = true;
}
if (mode == ActionType.NetworkServices)
{
//network service polling
//OCM.API.NetworkServices.ServiceManager serviceManager = new OCM.API.NetworkServices.ServiceManager();
//serviceManager.Test(OCM.API.NetworkServices.ServiceProvider.CoulombChargePoint);
}
#if DEBUG
if (mode == ActionType.ClientTest)
{
OCMClient client = new OCMClient();
client.APITestTiming();
actionPerformed = true;
}
#endif
if (!actionPerformed)
{
LogEvent("Nothing to do. Exiting. " + DateTime.UtcNow.ToShortTimeString());
}
}
}
public async Task <ImportReport> PerformImport(ExportType exportType, bool fetchLiveData, APICredentials credentials, CoreReferenceData coreRefData, string outputPath, IImportProvider provider)
{
var p = ((BaseImportProvider)provider);
p.ExportType = exportType;
ImportReport resultReport = new ImportReport();
resultReport.ProviderDetails = p;
try
{
bool loadOK = false;
if (fetchLiveData && p.IsAutoRefreshed && !String.IsNullOrEmpty(p.AutoRefreshURL))
{
p.Log("Loading input data from URL..");
loadOK = p.LoadInputFromURL(p.AutoRefreshURL);
}
else
{
if (p.IsStringData)
{
p.Log("Loading input data from file..");
loadOK = p.LoadInputFromFile(p.InputPath);
}
else
{
//binary streams pass as OK by default
loadOK = true;
}
}
if (!loadOK)
{
//failed to load
p.Log("Failed to load input data.");
throw new Exception("Failed to fetch input data");
}
List <ChargePoint> duplicatesList = new List <ChargePoint>();
p.Log("Processing input..");
var list = provider.Process(coreRefData);
int numAdded = 0;
int numUpdated = 0;
if (list.Count > 0)
{
p.Log("De-Deuplicating list (" + p.ProviderName + ":: " + list.Count + " Items)..");
//de-duplicate and clean list based on existing data
//TODO: take original and replace in final update list, setting relevant updated properties (merge) and status
var finalList = await DeDuplicateList(list, true, coreRefData, resultReport);
//var finalList = list;
if (ImportUpdatesOnly)
{
finalList = finalList.Where(l => l.ID > 0).ToList();
}
//finalList = client.GetLocations(new SearchFilters { MaxResults = 10000 });
//export/apply updates
if (p.ExportType == ExportType.XML)
{
p.Log("Exporting XML..");
//output xml
p.ExportXMLFile(finalList, outputPath + p.OutputNamePrefix + ".xml");
}
if (p.ExportType == ExportType.CSV)
{
p.Log("Exporting CSV..");
//output csv
p.ExportCSVFile(finalList, outputPath + p.OutputNamePrefix + ".csv");
}
if (p.ExportType == ExportType.JSON)
{
p.Log("Exporting JSON..");
//output json
p.ExportJSONFile(finalList, outputPath + p.OutputNamePrefix + ".json");
}
if (p.ExportType == ExportType.API && p.IsProductionReady)
{
//publish list of locations to OCM via API
OCMClient ocmClient = new OCMClient(IsSandboxedAPIMode);
p.Log("Publishing via API..");
foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.Country != null))
{
ocmClient.UpdateItem(cp, credentials);
if (cp.ID == 0)
{
numAdded++;
}
else
{
numUpdated++;
}
}
}
if (p.ExportType == ExportType.POIModelList)
{
resultReport.Added = finalList.Where(cp => cp.ID == 0).ToList();
resultReport.Updated = finalList.Where(cp => cp.ID > 0).ToList();
}
}
p.Log("Import Processed:" + provider.GetProviderName() + " Added:" + numAdded + " Updated:" + numUpdated);
}
catch (Exception exp)
{
p.Log("Import Failed:" + provider.GetProviderName() + " ::" + exp.ToString());
}
return(resultReport);
}
public async Task <ImportReport> PerformImport(ExportType exportType, bool fetchLiveData, APICredentials credentials, CoreReferenceData coreRefData, string outputPath, IImportProvider provider, bool cacheInputData, bool fetchExistingFromAPI = false)
{
var p = ((BaseImportProvider)provider);
p.ExportType = exportType;
ImportReport resultReport = new ImportReport();
resultReport.ProviderDetails = p;
try
{
bool loadOK = false;
if (p.ImportInitialisationRequired && p is IImportProviderWithInit)
{
((IImportProviderWithInit)provider).InitImportProvider();
}
if (fetchLiveData && p.IsAutoRefreshed && !String.IsNullOrEmpty(p.AutoRefreshURL))
{
Log("Loading input data from URL..");
loadOK = p.LoadInputFromURL(p.AutoRefreshURL);
}
else
{
if (p.IsStringData && !p.UseCustomReader)
{
Log("Loading input data from file..");
loadOK = p.LoadInputFromFile(p.InputPath);
}
else
{
//binary streams pass as OK by default
loadOK = true;
}
}
if (!loadOK)
{
//failed to load
Log("Failed to load input data.");
throw new Exception("Failed to fetch input data");
}
else
{
if (fetchLiveData && cacheInputData)
{
//save input data
p.SaveInputFile(p.InputPath);
}
}
List <ChargePoint> duplicatesList = new List <ChargePoint>();
Log("Processing input..");
var list = provider.Process(coreRefData);
int numAdded = 0;
int numUpdated = 0;
if (list.Count > 0)
{
if (p.MergeDuplicatePOIEquipment)
{
Log("Merging Equipment from Duplicate POIs");
list = MergeDuplicatePOIEquipment(list);
}
if (!p.IncludeInvalidPOIs)
{
Log("Cleaning invalid POIs");
var invalidPOIs = new List <ChargePoint>();
foreach (var poi in list)
{
if (!BaseImportProvider.IsPOIValidForImport(poi))
{
invalidPOIs.Add(poi);
}
}
foreach (var poi in invalidPOIs)
{
list.Remove(poi);
}
}
GC.Collect();
List <ChargePoint> finalList = new List <ChargePoint>();
if (!p.SkipDeduplication)
{
Log("De-Deuplicating list (" + p.ProviderName + ":: " + list.Count + " Items)..");
//de-duplicate and clean list based on existing data
//TODO: take original and replace in final update list, setting relevant updated properties (merge) and status
finalList = await DeDuplicateList(list.ToList(), true, coreRefData, resultReport, p.AllowDuplicatePOIWithDifferentOperator, fetchExistingFromAPI);
//var finalList = list;
}
else
{
//skip deduplication
finalList = list.ToList();
}
if (ImportUpdatesOnly)
{
finalList = finalList.Where(l => l.ID > 0).ToList();
}
//finalList = client.GetLocations(new SearchFilters { MaxResults = 10000 });
GC.Collect();
//export/apply updates
if (p.ExportType == ExportType.XML)
{
Log("Exporting XML..");
//output xml
p.ExportXMLFile(finalList, outputPath + p.OutputNamePrefix + ".xml");
}
if (p.ExportType == ExportType.CSV)
{
Log("Exporting CSV..");
//output csv
p.ExportCSVFile(finalList, outputPath + p.OutputNamePrefix + ".csv");
}
if (p.ExportType == ExportType.JSON)
{
Log("Exporting JSON..");
//output json
p.ExportJSONFile(finalList, outputPath + p.OutputNamePrefix + ".json");
}
if (p.ExportType == ExportType.API && p.IsProductionReady)
{
//publish list of locations to OCM via API
OCMClient ocmClient = new OCMClient(IsSandboxedAPIMode);
Log("Publishing via API..");
foreach (ChargePoint cp in finalList.Where(l => l.AddressInfo.Country != null))
{
ocmClient.UpdateItem(cp, credentials);
if (cp.ID == 0)
{
numAdded++;
}
else
{
numUpdated++;
}
}
}
if (p.ExportType == ExportType.POIModelList)
{
//result report contains POI lists
}
}
Log("Import Processed:" + provider.GetProviderName() + " Added:" + numAdded + " Updated:" + numUpdated);
}
catch (Exception exp)
{
Log("Import Failed:" + provider.GetProviderName() + " ::" + exp.ToString());
}
resultReport.Log = "";
resultReport.Log += p.ProcessingLog;
resultReport.Log += ImportLog;
return(resultReport);
}
