public void CleanupRedundantEditQueueitems()
{
var sourceList = DataModel.EditQueueItems;
var redundantEdits = new List<Model.EditQueueItem>();
var cpManager = new POIManager();
foreach (var item in sourceList)
{
var editItem = GetItemWithDifferences(item, cpManager, true);
if (editItem.Differences.Count == 0)
{
redundantEdits.Add(editItem);
}
}
//delete redundant edits
foreach (var item in redundantEdits)
{
var delItem = DataModel.EditQueueItems.Find(item.ID);
DataModel.EditQueueItems.Remove(delItem);
}
DataModel.SaveChanges();
}
public Model.EditQueueItem GetItemWithDifferences(Core.Data.EditQueueItem item, POIManager cpManager, bool loadCurrentItem)
{
var queueItem = Model.Extensions.EditQueueItem.FromDataModel(item);
//get diff between previous and edit
Model.ChargePoint poiA = DeserializePOIFromJSON(queueItem.PreviousData);
if (loadCurrentItem && poiA != null)
{
poiA = new POIManager().Get(poiA.ID);
}
Model.ChargePoint poiB = DeserializePOIFromJSON(queueItem.EditData);
queueItem.Differences = cpManager.CheckDifferences(poiA, poiB, useObjectCompare: true);
return queueItem;
}
protected void Page_Load(object sender, EventArgs e)
{
var evseManager = new POIManager();
var testPoint = evseManager.Get(3400);
if (testPoint != null)
{
List<ChargePoint> sourceList = new List<ChargePoint>();
sourceList.Add(testPoint);
GridViewSource.DataSource = sourceList;
GridViewSource.DataBind();
var similarPoints = evseManager.FindSimilar(testPoint);
GridViewSimilar.DataSource = similarPoints;
GridViewSimilar.DataBind();
}
else
{
}
}
public List<Model.EditQueueItem> GetEditQueueItems(EditQueueFilter filter)
{
var sourceList =
DataModel.EditQueueItems.Where(
i => (
(filter.ShowProcessed || (filter.ShowProcessed == false && i.IsProcessed == false))
&& (filter.DateFrom == null || (filter.DateFrom != null && i.DateSubmitted >= filter.DateFrom))
&& (filter.DateTo == null || (filter.DateTo != null && i.DateSubmitted <= filter.DateTo))
&& (filter.ShowEditsOnly == false || (filter.ShowEditsOnly == true && i.PreviousData != null))
)).OrderByDescending(e => e.DateSubmitted);
var cpManager = new POIManager();
var outputList = new List<Model.EditQueueItem>();
//perform object level differencing on json contents of edit queue items (very expensive), used to get summary and count of differences per item
foreach (var editQueueItem in sourceList)
{
outputList.Add(GetItemWithDifferences(editQueueItem, cpManager, false));
}
return outputList.Where(i => i.Differences.Count >= filter.MinimumDifferences).Take(filter.MaxResults).ToList();
}
public MediaItem AddPOIMediaItem(string tempFolder, string sourceImageFile, int chargePointId, string comment, bool isVideo, int userId)
{
var dataModel = new OCMEntities();
var mediaItem = new MediaItem();
var poi = new POIManager().Get(chargePointId);
string[] urls = UploadPOIImageToStorage(tempFolder, sourceImageFile, poi);
if (urls == null)
{
//failed to upload, preserve submission data
System.IO.File.WriteAllText(tempFolder + "//OCM_" + chargePointId + "_" + (DateTime.Now.ToFileTimeUtc().ToString()) + ".json", "{userId:" + userId + ",comment:\"" + comment + "\"}");
return null;
}
else
{
mediaItem.ItemURL = urls[0];
mediaItem.ItemThumbnailURL = urls[1];
mediaItem.User = dataModel.Users.FirstOrDefault(u => u.ID == userId);
mediaItem.ChargePoint = dataModel.ChargePoints.FirstOrDefault(cp => cp.ID == chargePointId);
mediaItem.Comment = comment;
mediaItem.DateCreated = DateTime.UtcNow;
mediaItem.IsEnabled = true;
mediaItem.IsExternalResource = false;
mediaItem.IsVideo = isVideo;
dataModel.MediaItems.Add(mediaItem);
dataModel.ChargePoints.Find(chargePointId).DateLastStatusUpdate = DateTime.UtcNow;
dataModel.SaveChanges();
new UserManager().AddReputationPoints(userId, 1);
return mediaItem;
}
}
public POIDuplicates GetAllPOIDuplicates(POIManager poiManager, int countryId, double maxDupeRange = 500)
{
List<DuplicatePOIItem> allDuplicates = new List<DuplicatePOIItem>();
var dataModel = new OCM.Core.Data.OCMEntities();
double DUPLICATE_DISTANCE_METERS = 25;
double POSSIBLE_DUPLICATE_DISTANCE_METERS = maxDupeRange;
//TODO: better method for large number of POIs
//grab all live POIs (30-100,000 items)
//var allPOIs = dataModel.ChargePoints.Where(s => s.AddressInfo.CountryID == countryId && (s.SubmissionStatusTypeID == 100 || s.SubmissionStatusTypeID == 200)).ToList();
var allPOIs = poiManager.GetChargePoints(new APIRequestParams { CountryIDs = new int[] { countryId }, MaxResults=100000 });
foreach (var poi in allPOIs)
{
//find pois which duplicate the given poi
var dupePOIs = allPOIs.Where(p => p.ID != poi.ID &&
(
p.DataProvidersReference != null && p.DataProvidersReference.Length > 0 && p.DataProvidersReference == poi.DataProvidersReference
|| new System.Device.Location.GeoCoordinate(p.AddressInfo.Latitude, p.AddressInfo.Longitude).GetDistanceTo(new System.Device.Location.GeoCoordinate(poi.AddressInfo.Latitude, poi.AddressInfo.Longitude)) < POSSIBLE_DUPLICATE_DISTANCE_METERS
)
);
if (dupePOIs.Any())
{
var poiModel = poi;// OCM.API.Common.Model.Extensions.ChargePoint.FromDataModel(poi, true, true, true, true);
foreach (var dupe in dupePOIs)
{
//poi has duplicates
DuplicatePOIItem dupePoi = new DuplicatePOIItem { DuplicatePOI = dupe, DuplicateOfPOI = poiModel };
dupePoi.Reasons = new List<string>();
if (dupe.AddressInfo.Latitude == poi.AddressInfo.Latitude && dupe.AddressInfo.Longitude == poi.AddressInfo.Longitude)
{
dupePoi.Reasons.Add("POI location is exact match for OCM-" + poi.ID);
dupePoi.Confidence = 95;
}
else
{
double distanceMeters = new System.Device.Location.GeoCoordinate(dupe.AddressInfo.Latitude, dupe.AddressInfo.Longitude).GetDistanceTo(new System.Device.Location.GeoCoordinate(poi.AddressInfo.Latitude, poi.AddressInfo.Longitude));
if (distanceMeters < DUPLICATE_DISTANCE_METERS)
{
dupePoi.Reasons.Add("POI location is close proximity (" + distanceMeters + "m) to OCM-" + poi.ID);
dupePoi.Confidence = 75;
}
else
{
if (distanceMeters < POSSIBLE_DUPLICATE_DISTANCE_METERS)
{
dupePoi.Reasons.Add("POI location is in surrounding proximity (" + distanceMeters + "m) to OCM-" + poi.ID);
dupePoi.Confidence = 50;
}
}
}
allDuplicates.Add(dupePoi);
}
}
}
//arrange all duplicates into groups
POIDuplicates duplicatesSummary = new POIDuplicates();
duplicatesSummary.DuplicateSummaryList = new List<DuplicatePOIGroup>();
foreach (var dupe in allDuplicates)
{
bool isNewGroup = false;
var dupeGroup = duplicatesSummary.DuplicateSummaryList.FirstOrDefault(d => d.DuplicatePOIList.Any(p => p.DuplicateOfPOI.ID == dupe.DuplicateOfPOI.ID || p.DuplicatePOI.ID == dupe.DuplicatePOI.ID) || d.SuggestedBestPOI.ID == dupe.DuplicatePOI.ID);
if (dupeGroup == null)
{
isNewGroup = true;
dupeGroup = new DuplicatePOIGroup();
dupeGroup.SuggestedBestPOI = dupe.DuplicatePOI;//TODO: select best
dupeGroup.DuplicatePOIList = new List<DuplicatePOIItem>();
}
//only add to dupe group if not already added for another reason
if (!dupeGroup.DuplicatePOIList.Contains(dupe) && !dupeGroup.DuplicatePOIList.Any(d => d.DuplicatePOI.ID == dupe.DuplicatePOI.ID))
{
dupeGroup.DuplicatePOIList.Add(dupe);
}
if (isNewGroup)
{
duplicatesSummary.DuplicateSummaryList.Add(dupeGroup);
}
}
//loop through groups and rearrange
RearrangeDuplicates(duplicatesSummary);
//go through all groups and populate final list of All POI per group
foreach (var g in duplicatesSummary.DuplicateSummaryList)
{
var poiList = new List<OCM.API.Common.Model.ChargePoint>();
foreach (var d in g.DuplicatePOIList)
{
if (!poiList.Contains(d.DuplicatePOI))
{
poiList.Add(d.DuplicatePOI);
}
if (!poiList.Contains(d.DuplicateOfPOI))
{
poiList.Add(d.DuplicateOfPOI);
}
g.AllPOI = poiList;
}
}
//TODO: go through all dupe groups and nominate best poi to be main poi (most comments, most equipment info etc)
return duplicatesSummary;
}
public ActionResult Edit(ChargePoint poi)
{
var refData = new POIBrowseModel();
refData.AllowOptionalCountrySelection = false;
ViewBag.ReferenceData = refData;
ViewBag.ConnectionIndex = 0; //connection counter shared by equipment details
ViewBag.EnableEditView = true;
if (Request["editoption"] == "addconnection")
{
//add a placeholder for new equipment details
if (poi.Connections == null) poi.Connections = new List<ConnectionInfo>();
//TODO: setup defaults
poi.Connections.Add(new ConnectionInfo());
return View(poi);
}
if (Request["editoption"].ToString().StartsWith("remove-equipment"))
{
//TODO:remove requested connection
//poi.Connections.Remove();
string[] equipmentElementIDs = Request["editoption"].ToString().Split('-');
int itemIndex = int.Parse(equipmentElementIDs[2]);
poi.Connections.RemoveAt(itemIndex);
return View(poi);
}
if (Request["editoption"] == "preview")
{
//preview poi
ViewBag.EnablePreviewMode = true;
//reset any values provided as -1 to a standard default (unknown etc)
PrepareDefaultsForBlankSelections(poi);
//update preview of poi with fully populated reference data
poi = new POIManager().PreviewPopulatedPOIFromModel(poi);
InitEditReferenceData(poi);
return View(poi);
}
if (ModelState.IsValid)
{
try
{
User user = null;
if (IsUserSignedIn) user = new UserManager().GetUser((int)Session["UserID"]);
//reset any values provided as -1 to a standard default (unknown etc)
PrepareDefaultsForBlankSelections(poi);
if (poi.AddressInfo.Country == null || poi.AddressInfo.Country.ID == -1) ModelState.AddModelError("Country", "Required");
//perform actual POI submission, then redirect to POI details if we can
int poiSubmissionID = new SubmissionManager().PerformPOISubmission(poi, user);
if (poiSubmissionID > -1)
{
if (poiSubmissionID > 0)
{
return RedirectToAction("Details", "POI", new { id = poiSubmissionID, status = "editsubmitted" });
}
else
{
return RedirectToAction("Index");
}
}
else
{
ViewBag.ValidationFailed = true;
}
}
catch
{
//return View(poi);
}
}
else
{
foreach (ModelState modelState in ViewData.ModelState.Values)
{
foreach (ModelError error in modelState.Errors)
{
System.Diagnostics.Debug.WriteLine(error.ToString());
}
}
}
ViewBag.ReferenceData = new POIBrowseModel();
return View(poi);
}
//
// GET: /POI/Edit/5
public ActionResult Edit(int? id, bool createCopy = false)
{
if (id > 0)
{
ChargePoint poi = null;
if (createCopy)
{
//get version of POI with location details removed, copying equipment etc
poi = new POIManager().GetCopy((int)id, true);
}
else
{
poi = new POIManager().Get((int)id);
}
if (poi != null)
{
InitEditReferenceData(poi);
var refData = new POIBrowseModel();
ViewBag.ReferenceData = refData;
ViewBag.HideAdvancedInfo = true;
if (!createCopy)
{
try
{
var user = new UserManager().GetUser((int)Session["UserID"]);
if (POIManager.CanUserEditPOI(poi, user))
{
ViewBag.HideAdvancedInfo = false;
}
}
catch (Exception)
{
; ; //user not signed in
}
}
//enable advanced edit options for full editors/admin
return View(poi);
}
}
//no applicable poi, jump back to browse
return RedirectToAction("Index", "POI");
}
/// <summary>
/// Output compact (id's only for reference data) POI list
/// </summary>
/// <param name="context"></param>
/// <param name="filter"></param>
private void OutputCompactPOIList(HttpContext context, APIRequestParams filter)
{
//get list of charge points as compact POISearchResult List for output:
List<OCM.API.Common.Model.ChargePoint> dataList = new POIManager().GetChargePoints(filter);
int numResults = dataList.Count;
List<POISearchResult> poiList = new List<POISearchResult>();
foreach (var c in dataList)
{
var poi = new POISearchResult
{
ID = c.ID,
Title = c.AddressInfo.Title,
Address = c.AddressInfo.AddressLine1,
Distance = c.AddressInfo.Distance,
Latitude = (double)c.AddressInfo.Latitude,
Longitude = (double)c.AddressInfo.Longitude,
Postcode = c.AddressInfo.Postcode,
UsageTypeID = c.UsageType != null ? c.UsageType.ID : 0,
StatusTypeID = c.StatusType != null ? c.StatusType.ID : 0
};
poiList.Add(poi);
}
var o = new JSONOutputProvider();
string description = "Compact POI List: id=ID, t= Title, u= UsageTypeID, s= StatusTypeID, a=Address, p=Postcode, lt=Latitude, lg=Longitude, d= Distance";
o.PerformSerialisationV2(context.Response.OutputStream, new { status = "OK", description = description, count = poiList.Count, results = poiList }, filter.Callback);
}
/// <summary>
/// Output standard POI List results
/// </summary>
/// <param name="outputProvider"></param>
/// <param name="context"></param>
/// <param name="filter"></param>
private void OutputPOIList(IOutputProvider outputProvider, HttpContext context, APIRequestParams filter)
{
List<OCM.API.Common.Model.ChargePoint> dataList = null;
//get list of charge points for output:
dataList = new POIManager().GetChargePoints(filter);
int numResults = dataList.Count;
//send response
outputProvider.GetOutput(context.Response.OutputStream, dataList, filter);
}
/**
* Generic Import Process
Provider Properties
Import Method
Import URL/Path
Import Frequency
IsMaster
Fetch Latest Data
For each item
Check If Exists or Strong Duplicate, Get ID
If New, Add
if Exists Then
Prepare update, if provider supports live status, set that
What if item updated manually on OCM?
Send Update
End
Loop
Log Exceptions
Log Count of Items Added or Modified
Way to remove item (or log items) which no longer exist in data source?
* */
public async Task<List<ChargePoint>> DeDuplicateList(List<ChargePoint> cpList, bool updateDuplicate, CoreReferenceData coreRefData, ImportReport report, bool allowDupeWithDifferentOperator = false)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
//get list of all current POIs (in relevant countries) including most delisted ones
int[] countryIds = (from poi in cpList
where poi.AddressInfo.Country != null
select poi.AddressInfo.Country.ID).Distinct().ToArray();
APIRequestParams filters = new APIRequestParams { CountryIDs = countryIds, MaxResults = 1000000, EnableCaching = false, SubmissionStatusTypeID = 0 };
//List<ChargePoint> masterList = await new OCMClient(IsSandboxedAPIMode).GetLocations(filters); //new OCMClient().FindSimilar(null, 10000); //fetch all charge points regardless of status
var poiManager = new POIManager();
List<ChargePoint> masterListCollection = poiManager.GetChargePoints(filters); //new OCMClient().FindSimilar(null, 10000); //fetch all charge points regardless of status
var spec = new i4o.IndexSpecification<ChargePoint>()
.Add(i => i.DataProviderID)
.Add(i => i.DataProvidersReference)
;
var masterList = new i4o.IndexSet<ChargePoint>(masterListCollection, spec);
List<ChargePoint> masterListCopy = new List<ChargePoint>();
foreach (var tmp in masterList)
{
//fully copy of master list item so we have before/after
masterListCopy.Add(JsonConvert.DeserializeObject<ChargePoint>(JsonConvert.SerializeObject(tmp)));
}
//if we failed to get a master list, quit with no result
if (masterListCollection.Count == 0) return new List<ChargePoint>();
List<ChargePoint> duplicateList = new List<ChargePoint>();
List<ChargePoint> updateList = new List<ChargePoint>();
ChargePoint previousCP = null;
//for each item to be imported, deduplicate by adding to updateList only the items which we don't already haves
var cpListSortedByPos = cpList.OrderBy(c => c.AddressInfo.Latitude).ThenBy(c => c.AddressInfo.Longitude);
int poiProcessed = 0;
int totalPOI = cpListSortedByPos.Count();
Stopwatch dupeIdentWatch = new Stopwatch();
dupeIdentWatch.Start();
foreach (var item in cpListSortedByPos)
{
var itemGeoPos = new System.Device.Location.GeoCoordinate(item.AddressInfo.Latitude, item.AddressInfo.Longitude);
//item is duplicate if we already seem to have it based on Data Providers reference or approx position match
var dupeList = masterList.Where(c =>
(c.DataProvider != null && c.DataProviderID == item.DataProviderID && c.DataProvidersReference == item.DataProvidersReference)
|| (c.AddressInfo.Title == item.AddressInfo.Title && c.AddressInfo.AddressLine1 == item.AddressInfo.AddressLine1 && c.AddressInfo.Postcode == item.AddressInfo.Postcode)
|| (GeoManager.IsClose(c.AddressInfo.Latitude, c.AddressInfo.Longitude, item.AddressInfo.Latitude, item.AddressInfo.Longitude) && new System.Device.Location.GeoCoordinate(c.AddressInfo.Latitude, c.AddressInfo.Longitude).GetDistanceTo(itemGeoPos) < DUPLICATE_DISTANCE_METERS) //meters distance apart
);
if (dupeList.Any())
{
if (updateDuplicate)
{
//if updating duplicates, get exact matching duplicate based on provider reference and update/merge with this item to update status/merge properties
var updatedItem = dupeList.FirstOrDefault(d => d.DataProviderID == (item.DataProvider != null ? item.DataProvider.ID : item.DataProviderID) && d.DataProvidersReference == item.DataProvidersReference);
if (updatedItem != null)
{
//only merge/update from live published items
if (updatedItem.SubmissionStatus.IsLive == (bool?)true
|| updatedItem.SubmissionStatus.ID == (int)StandardSubmissionStatusTypes.Delisted_RemovedByDataProvider
|| updatedItem.SubmissionStatus.ID == (int)StandardSubmissionStatusTypes.Delisted_NotPublicInformation)
{
//item is an exact match from same data provider
//overwrite existing with imported data (use import as master)
//updatedItem = poiManager.PreviewPopulatedPOIFromModel(updatedItem);
MergeItemChanges(item, updatedItem, false);
updateList.Add(updatedItem);
}
}
if (updatedItem == null)
{
//duplicates are not exact match
//TODO: resolve whether imported data should change duplicate
//merge new properties from imported item
//if (item.StatusType != null) updatedItem.StatusType = item.StatusType;
//updateList.Add(updatedItem);
}
}
//item has one or more likely duplicates, add it to list of items to remove
duplicateList.Add(item);
}
//mark item as duplicate if location/title exactly matches previous entry or lat/long is within DuplicateDistance meters
if (previousCP != null)
{
//this branch is the most expensive part of dedupe:
if (IsDuplicateLocation(item, previousCP, true))
{
if (!duplicateList.Contains(item))
{
if (allowDupeWithDifferentOperator && item.OperatorID != previousCP.OperatorID)
{
Log("Duplicated allowed due to different operator:" + item.AddressInfo.Title);
}
else
{
Log("Duplicated item removed:" + item.AddressInfo.Title);
duplicateList.Add(item);
}
}
}
}
previousCP = item;
poiProcessed++;
if (poiProcessed % 300 == 0)
{
System.Diagnostics.Debug.WriteLine("Deduplication: " + poiProcessed + " processed of " + totalPOI);
}
}
dupeIdentWatch.Stop();
Log("De-dupe pass took " + dupeIdentWatch.Elapsed.TotalSeconds + " seconds. " + (dupeIdentWatch.Elapsed.TotalMilliseconds / cpList.Count) + "ms per item.");
//remove duplicates from list to apply
foreach (var dupe in duplicateList)
{
cpList.Remove(dupe);
}
Log("Duplicates removed from import:" + duplicateList.Count);
//add updated items (replace duplicates with property changes)
foreach (var updatedItem in updateList)
{
if (!cpList.Contains(updatedItem))
{
cpList.Add(updatedItem);
}
}
Log("Updated items to import:" + updateList.Count);
//populate missing location info from geolocation cache if possible
Stopwatch geoWatch = new Stopwatch();
geoWatch.Start();
PopulateLocationFromGeolocationCache(cpList, coreRefData);
geoWatch.Stop();
Log("Populate Country from Lat/Long took " + geoWatch.Elapsed.TotalSeconds + " seconds. " + (geoWatch.Elapsed.TotalMilliseconds / cpList.Count) + "ms per item.");
//final pass to catch duplicates present in data source, mark additional items as Delisted Duplicate so we have a record for them
var submissionStatusDelistedDupe = coreRefData.SubmissionStatusTypes.First(s => s.ID == 1001); //delisted duplicate
previousCP = null;
//sort current cp list by position again
cpListSortedByPos = cpList.OrderBy(c => c.AddressInfo.Latitude).ThenBy(c => c.AddressInfo.Longitude);
//mark any duplicates in final list as delisted duplicates (submitted to api)
foreach (var cp in cpListSortedByPos)
{
bool isDuplicate = false;
if (previousCP != null)
{
isDuplicate = IsDuplicateLocation(cp, previousCP, false);
if (isDuplicate)
{
cp.SubmissionStatus = submissionStatusDelistedDupe;
cp.SubmissionStatusTypeID = submissionStatusDelistedDupe.ID;
if (previousCP.ID > 0)
{
if (cp.GeneralComments == null) cp.GeneralComments = "";
cp.GeneralComments += " [Duplicate of OCM-" + previousCP.ID + "]";
cp.ParentChargePointID = previousCP.ID;
}
}
}
if (!isDuplicate)
{
previousCP = cp;
}
}
report.Added = cpListSortedByPos.Where(cp => cp.ID == 0).ToList();
report.Updated = cpListSortedByPos.Where(cp => cp.ID > 0).ToList();
report.Duplicates = duplicateList; //TODO: add additional pass of duplicates from above
//determine which POIs in our master list are no longer referenced in the import
report.Delisted = masterList.Where(cp => cp.DataProviderID == report.ProviderDetails.DataProviderID && cp.SubmissionStatus != null && (cp.SubmissionStatus.IsLive == true || cp.SubmissionStatusTypeID == (int)StandardSubmissionStatusTypes.Imported_UnderReview)
&& !cpListSortedByPos.Any(master => master.ID == cp.ID) && !report.Duplicates.Any(master => master.ID == cp.ID)
&& cp.UserComments == null && cp.MediaItems == null).ToList();
//safety check to ensure we're not delisting items just because we have incomplete import data:
if (cpList.Count < 50)// || (report.Delisted.Count > cpList.Count))
{
report.Delisted = new List<ChargePoint>();
}
//determine list of low quality POIs (incomplete address info etc)
report.LowDataQuality = new List<ChargePoint>();
report.LowDataQuality.AddRange(GetLowDataQualityPOIs(report.Added));
report.LowDataQuality.AddRange(GetLowDataQualityPOIs(report.Updated));
Log("Removing " + report.LowDataQuality.Count + " low quality POIs from added/updated");
//remove references in added/updated to any low quality POIs
foreach (var p in report.LowDataQuality)
{
report.Added.Remove(p);
}
foreach (var p in report.LowDataQuality)
{
report.Updated.Remove(p);
}
//remove updates which only change datelaststatusupdate
var updatesToIgnore = new List<ChargePoint>();
foreach (var poi in report.Updated)
{
var origPOI = masterListCopy.FirstOrDefault(p => p.ID == poi.ID);
var updatedPOI = poiManager.PreviewPopulatedPOIFromModel(poi);
var differences = poiManager.CheckDifferences(origPOI, updatedPOI);
differences.RemoveAll(d => d.Context == ".MetadataValues");
differences.RemoveAll(d => d.Context == ".DateLastStatusUpdate");
differences.RemoveAll(d => d.Context == ".UUID");
differences.RemoveAll(d => d.Context == ".DataProvider.DateLastImported");
differences.RemoveAll(d => d.Context == ".IsRecentlyVerified");
differences.RemoveAll(d => d.Context == ".DateLastVerified");
differences.RemoveAll(d => d.Context == ".UserComments");
differences.RemoveAll(d => d.Context == ".MediaItems");
if (!differences.Any())
{
updatesToIgnore.Add(poi);
}
else
{
//differences exist
CompareLogic compareLogic = new CompareLogic();
compareLogic.Config.MaxDifferences = 100;
compareLogic.Config.IgnoreObjectTypes = false;
compareLogic.Config.IgnoreUnknownObjectTypes = true;
compareLogic.Config.CompareChildren = true;
ComparisonResult result = compareLogic.Compare(origPOI, updatedPOI);
var diffReport = new KellermanSoftware.CompareNetObjects.Reports.UserFriendlyReport();
result.Differences.RemoveAll(d => d.PropertyName == ".MetadataValues");
result.Differences.RemoveAll(d => d.PropertyName == ".DateLastStatusUpdate");
result.Differences.RemoveAll(d => d.PropertyName == ".UUID");
result.Differences.RemoveAll(d => d.PropertyName == ".DataProvider.DateLastImported");
result.Differences.RemoveAll(d => d.PropertyName == ".IsRecentlyVerified");
result.Differences.RemoveAll(d => d.PropertyName == ".DateLastVerified");
result.Differences.RemoveAll(d => d.PropertyName == ".UserComments");
result.Differences.RemoveAll(d => d.PropertyName == ".MediaItems");
System.Diagnostics.Debug.WriteLine("Difference:" + diffReport.OutputString(result.Differences));
if (!result.Differences.Any())
{
updatesToIgnore.Add(poi);
}
}
}
foreach (var p in updatesToIgnore)
{
if (report.Unchanged == null) report.Unchanged = new List<ChargePoint>();
report.Unchanged.Add(p);
report.Updated.Remove(p);
}
//TODO: if POi is a duplicate ensure imported data provider reference/URL is included as reference metadata in OCM's version of the POI
stopWatch.Stop();
Log("Deduplicate List took " + stopWatch.Elapsed.TotalSeconds + " seconds");
//return final processed list ready for applying as insert/updates
return cpListSortedByPos.ToList();
}
//convert a simple POI to data and back again to fully populate all related properties, as submission may only have simple IDs for ref data etc
private Model.ChargePoint PopulateFullPOI(Model.ChargePoint poi)
{
OCMEntities tempDataModel = new OCMEntities();
//convert simple poi to fully populated db version
var poiData = new POIManager().PopulateChargePoint_SimpleToData(poi, tempDataModel);
//convert back to simple POI
var modelPOI = Model.Extensions.ChargePoint.FromDataModel(poiData, false, false, true, true);
//clear temp changes from the poi
//dataModel.Entry(poiData).Reload();
tempDataModel.Dispose();
return modelPOI;
}
/// <summary>
/// Consumers should prepare a new/updated ChargePoint with as much info populated as possible
/// </summary>
/// <param name="submission">ChargePoint info for submission, if ID and UUID set will be treated as an update</param>
/// <returns>false on error or not enough data supplied</returns>
public int PerformPOISubmission(Model.ChargePoint updatedPOI, Model.User user, bool performCacheRefresh = true, bool disablePOISuperseding = false)
{
try
{
var poiManager = new POIManager();
bool enableEditQueueLogging = bool.Parse(ConfigurationManager.AppSettings["EnableEditQueue"]);
bool isUpdate = false;
bool userCanEditWithoutApproval = false;
bool isSystemUser = false;
int? supersedesID = null;//POI from another data provider which has been superseded by an edit
//if user signed in, check if they have required permission to perform an edit/approve (if required)
if (user != null)
{
if (user.ID == (int)StandardUsers.System) isSystemUser = true;
//if user is system user, edits/updates are not recorded in edit queue
if (isSystemUser)
{
enableEditQueueLogging = false;
}
userCanEditWithoutApproval = POIManager.CanUserEditPOI(updatedPOI, user);
}
var dataModel = new Core.Data.OCMEntities();
//if poi is an update, validate if update can be performed
if (updatedPOI.ID > 0 && !String.IsNullOrEmpty(updatedPOI.UUID))
{
if (dataModel.ChargePoints.Any(c => c.ID == updatedPOI.ID && c.UUID == updatedPOI.UUID))
{
//update is valid poi, check if user has permission to perform an update
isUpdate = true;
if (userCanEditWithoutApproval) AllowUpdates = true;
if (!AllowUpdates && !enableEditQueueLogging)
{
return -1; //valid update requested but updates not allowed
}
}
else
{
//update does not correctly identify an existing poi
return -1;
}
}
//validate if minimal required data is present
if (updatedPOI.AddressInfo.Title == null || (updatedPOI.AddressInfo.Country == null && updatedPOI.AddressInfo.CountryID == null))
{
return -1;
}
//convert to DB version of POI and back so that properties are fully populated
updatedPOI = PopulateFullPOI(updatedPOI);
Model.ChargePoint oldPOI = null;
if (updatedPOI.ID > 0)
{
//get json snapshot of current cp data to store as 'previous'
oldPOI = poiManager.Get(updatedPOI.ID);
}
//if user cannot edit directly, add to edit queue for approval
var editQueueItem = new Core.Data.EditQueueItem { DateSubmitted = DateTime.UtcNow };
if (enableEditQueueLogging)
{
editQueueItem.EntityID = updatedPOI.ID;
editQueueItem.EntityType = dataModel.EntityTypes.FirstOrDefault(t => t.ID == 1);
//charging point location entity type id
//serialize cp as json
//null extra data we don't want to serialize/compare
updatedPOI.UserComments = null;
updatedPOI.MediaItems = null;
string editData = PerformSerialisationToString(updatedPOI, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
editQueueItem.EditData = editData;
if (updatedPOI.ID > 0)
{
//check if poi will change with this edit, if not we discard it completely
if (!poiManager.HasDifferences(oldPOI, updatedPOI))
{
System.Diagnostics.Debug.WriteLine("POI Update has no changes, discarding change.");
return updatedPOI.ID;
}
else
{
editQueueItem.PreviousData = PerformSerialisationToString(oldPOI, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
}
}
if (user != null) editQueueItem.User = dataModel.Users.FirstOrDefault(u => u.ID == user.ID);
editQueueItem.IsProcessed = false;
editQueueItem = dataModel.EditQueueItems.Add(editQueueItem);
//TODO: send notification of new item for approval
//save edit queue item
dataModel.SaveChanges();
//if previous edit queue item exists by same user for same POI, mark as processed
var previousEdits = dataModel.EditQueueItems.Where(e => e.UserID == editQueueItem.UserID && e.EntityID == editQueueItem.EntityID && e.EntityTypeID == editQueueItem.EntityTypeID && e.ID != editQueueItem.ID && e.IsProcessed != true);
foreach (var previousEdit in previousEdits)
{
previousEdit.IsProcessed = true;
if (editQueueItem.User != null)
{
previousEdit.ProcessedByUser = editQueueItem.User;
}
else
{
editQueueItem.ProcessedByUser = dataModel.Users.FirstOrDefault(u => u.ID == (int)StandardUsers.System);
}
previousEdit.DateProcessed = DateTime.UtcNow;
}
//save updated edit queue items
dataModel.SaveChanges();
}
//prepare and save changes POI changes/addition
if (isUpdate && !AllowUpdates)
{
//user has submitted an edit but is not an approved editor
//SendEditSubmissionNotification(updatedPOI, user);
//user is not an editor, item is now pending in edit queue for approval.
return updatedPOI.ID;
}
if (isUpdate && updatedPOI.SubmissionStatusTypeID >= 1000)
{
//update is a delisting, skip superseding poi
System.Diagnostics.Debug.WriteLine("skipping superseding of imported POI due to delisting");
}
else
{
//if poi being updated exists from an imported source we supersede the old POI with the new version, unless we're doing a fresh import from same data provider
if (!disablePOISuperseding)
{
//if update by non-system user will change an imported/externally provided data, supersede old POI with new one (retain ID against new POI)
if (isUpdate && !isSystemUser && oldPOI.DataProviderID != (int)StandardDataProviders.OpenChargeMapContrib)
{
//move old poi to new id, set status of new item to superseded
supersedesID = poiManager.SupersedePOI(dataModel, oldPOI, updatedPOI);
}
}
}
//user is an editor, go ahead and store the addition/update
//set/update cp properties
var cpData = poiManager.PopulateChargePoint_SimpleToData(updatedPOI, dataModel);
//if item has no submission status and user permitted to edit, set to published
if (userCanEditWithoutApproval && cpData.SubmissionStatusTypeID == null)
{
cpData.SubmissionStatusType = dataModel.SubmissionStatusTypes.First(s => s.ID == (int)StandardSubmissionStatusTypes.Submitted_Published);
cpData.SubmissionStatusTypeID = cpData.SubmissionStatusType.ID; //hack due to conflicting state change for SubmissionStatusType
}
else
{
//no submission status, set to 'under review'
if (cpData.SubmissionStatusType == null) cpData.SubmissionStatusType = dataModel.SubmissionStatusTypes.First(s => s.ID == (int)StandardSubmissionStatusTypes.Submitted_UnderReview);
}
cpData.DateLastStatusUpdate = DateTime.UtcNow;
if (!isUpdate)
{
//new data objects need added to data model before save
if (cpData.AddressInfo != null) dataModel.AddressInfoList.Add(cpData.AddressInfo);
dataModel.ChargePoints.Add(cpData);
}
//finally - save poi update
dataModel.SaveChanges();
//get id of update/new poi
int newPoiID = cpData.ID;
//this is an authorised update, reflect change in edit queue item
if (enableEditQueueLogging && user != null && user.ID > 0)
{
var editUser = dataModel.Users.FirstOrDefault(u => u.ID == user.ID);
editQueueItem.User = editUser;
if (newPoiID > 0) editQueueItem.EntityID = newPoiID;
//if user is authorised to edit, process item automatically without review
if (userCanEditWithoutApproval)
{
editQueueItem.ProcessedByUser = editUser;
editQueueItem.DateProcessed = DateTime.UtcNow;
editQueueItem.IsProcessed = true;
}
//save edit queue item changes
dataModel.SaveChanges();
}
else
{
//anonymous submission, update edit queue item
if (enableEditQueueLogging && user == null)
{
if (newPoiID > 0) editQueueItem.EntityID = newPoiID;
dataModel.SaveChanges();
}
}
System.Diagnostics.Debug.WriteLine("Added/Updated CP:" + cpData.ID);
//if user is not anonymous, log their submission and update their reputation points
if (user != null)
{
AuditLogManager.Log(user, isUpdate ? AuditEventType.UpdatedItem : AuditEventType.CreatedItem, "Modified OCM-" + cpData.ID, null);
//add reputation points
new UserManager().AddReputationPoints(user, 1);
}
//preserve new POI Id for caller
updatedPOI.ID = cpData.ID;
if (performCacheRefresh)
{
var cacheTask = Task.Run(async () =>
{
if (supersedesID != null)
{
await CacheManager.RefreshCachedPOI((int)supersedesID);
}
return await CacheManager.RefreshCachedPOI(updatedPOI.ID);
});
cacheTask.Wait();
}
return updatedPOI.ID;
}
catch (Exception exp)
{
System.Diagnostics.Debug.WriteLine(exp.ToString());
AuditLogManager.ReportWebException(HttpContext.Current.Server, AuditEventType.SystemErrorWeb);
//throw exp;
//error performing submission
return -1;
}
}
public void ProcessEditQueueItem(int id, bool publishEdit, int userId)
{
//prepare poi details
int updatePOIId = 0;
var queueItem = DataModel.EditQueueItems.FirstOrDefault(e => e.ID == id);
if (queueItem != null && queueItem.IsProcessed == false)
{
if (queueItem.EntityType.ID == (int)StandardEntityTypes.POI)
{
//check current user is authorized to approve edits for this POIs country
bool hasEditPermission = false;
var editPOI = DeserializePOIFromJSON(queueItem.EditData);
var userProfile = new UserManager().GetUser(userId);
if (userProfile != null)
{
if (UserManager.HasUserPermission(userProfile, editPOI.AddressInfo.CountryID, PermissionLevel.Editor))
{
hasEditPermission = true;
}
}
//processing a POI add/edit
if (hasEditPermission)
{
if (publishEdit)
{
//get diff between previous and edit
POIManager poiManager = new POIManager();
Model.ChargePoint poiA = DeserializePOIFromJSON(queueItem.PreviousData);
Model.ChargePoint poiB = DeserializePOIFromJSON(queueItem.EditData);
bool poiUpdateRequired = false;
if (poiA != null)
{
//this is an edit, load the latest version of the POI as version 'A'
poiA = poiManager.Get(poiA.ID);
if (poiManager.HasDifferences(poiA, poiB))
{
poiUpdateRequired = true;
}
}
//save poi update
//if its an edit, load the original details before applying the change
if (poiUpdateRequired)
{
//updates to externally provided POIs require old version to be superseded (archived) first
if (poiA != null && poiA.DataProviderID != (int)StandardDataProviders.OpenChargeMapContrib)
{
poiManager.SupersedePOI(DataModel, poiA, poiB);
}
}
//set/update cp properties from simple model to data model
var poiData = poiManager.PopulateChargePoint_SimpleToData(poiB, DataModel);
//set status type to published if previously unset
if (poiData.SubmissionStatusTypeID == null)
{
poiData.SubmissionStatusType = DataModel.SubmissionStatusTypes.First(s => s.ID == (int)StandardSubmissionStatusTypes.Submitted_Published);
}
poiData.DateLastStatusUpdate = DateTime.UtcNow;
//publish edit
DataModel.SaveChanges();
updatePOIId = poiData.ID;
//attribute submitter with reputation points
if (queueItem.UserID != null)
{
new UserManager().AddReputationPoints((int)queueItem.UserID, 1);
}
}
//update edit queue item as processed
queueItem.IsProcessed = true;
queueItem.ProcessedByUser = DataModel.Users.FirstOrDefault(u => u.ID == userId);
queueItem.DateProcessed = DateTime.UtcNow;
DataModel.SaveChanges();
//TODO: also award processing editor with reputation points if they are approving someone elses edit and they are not Admin
//Refresh POI cache
Task cacheRefresh = CacheManager.RefreshCachedPOI(updatePOIId);
}
}
}
}