private async Task <string?> ParseFacebookAsync(FarmData farmData)
{
if (string.IsNullOrWhiteSpace(farmData.SocialMedia))
{
return(null);
}
var parts = farmData.SocialMedia.Split(' ', ',');
string?url = parts.FirstOrDefault(p => p.Contains("facebook.com", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(url))
{
return(null);
}
if (!url.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
{
url = $"https://{url}";
}
try
{
var response = await HttpClient.GetAsync(url);
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
return(url);
}
}
catch (Exception)
{
}
return(null);
}
private void MergeDuplicates(List <FarmData> farms)
{
// Some rows are duplicated, merge them together to
// get the most information into a single record
var duplicateNames = farms.GroupBy(f => f.Name)
.Where(g => g.Count() > 1)
.ToDictionary(g => g.Key, g => g.ToList());
foreach (var duplicateName in duplicateNames)
{
FarmData merged = duplicateName.Value.First();
foreach (var duplicate in duplicateName.Value.Skip(1))
{
merged.Postcode = MostValuable(merged.Postcode, duplicate.Postcode);
merged.Town = MostValuable(merged.Town, duplicate.Town);
merged.County = MostValuable(merged.County, duplicate.County);
merged.SocialMedia = MostValuable(merged.SocialMedia, duplicate.SocialMedia);
merged.Product = MostValuable(merged.Product, duplicate.Product);
merged.Feature = MostValuable(merged.Feature, duplicate.Feature);
merged.Website = MostValuable(merged.Website, duplicate.Website);
merged.OnlineOrdering = MostValuable(merged.OnlineOrdering, duplicate.OnlineOrdering);
merged.Delivery = MostValuable(merged.Delivery, duplicate.Delivery);
merged.AcceptingNewCustomers = MostValuable(merged.AcceptingNewCustomers, duplicate.AcceptingNewCustomers);
merged.PickupHub = MostValuable(merged.PickupHub, duplicate.PickupHub);
merged.PreferredAccess = MostValuable(merged.PreferredAccess, duplicate.PreferredAccess);
merged.Contact = MostValuable(merged.Contact, duplicate.Contact);
merged.NeedVolunteers = MostValuable(merged.NeedVolunteers, duplicate.NeedVolunteers);
merged.AmpleSupply = MostValuable(merged.AmpleSupply, duplicate.AmpleSupply);
farms.Remove(duplicate);
}
}
}
private bool IsDifferent(FarmData farmData, Farm dbFarm)
{
return(farmData.Name != dbFarm.Name ||
farmData.Town != dbFarm.Town ||
farmData.County != dbFarm.County ||
farmData.Postcode != dbFarm.Postcode);
}
private async Task <string?> ParseWebsiteAsync(FarmData farmData)
{
if (string.IsNullOrWhiteSpace(farmData.SocialMedia))
{
return(null);
}
var parts = farmData.SocialMedia.Split(' ', ',');
string?address = parts.FirstOrDefault(p => p.Contains(".") && !p.Contains("facebook.com", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(address))
{
return(null);
}
string httpsAddress = address;
if (!address.StartsWith("http", StringComparison.InvariantCultureIgnoreCase))
{
httpsAddress = $"https://{address}";
}
try
{
var response = await HttpClient.GetAsync(httpsAddress);
response.EnsureSuccessStatusCode();
return(httpsAddress);
}
catch (Exception ex)
{
Logger.LogDebug(ex, $"Failed to query website on URL \"{httpsAddress}\" for farm \"{farmData.Name}\"");
}
if (!address.StartsWith("http", StringComparison.InvariantCultureIgnoreCase))
{
string httpAddress = $"http://{address}";
try
{
var response = await HttpClient.GetAsync(httpAddress);
response.EnsureSuccessStatusCode();
return(httpAddress);
}
catch (Exception ex)
{
Logger.LogDebug(ex, $"Failed to query website on URL \"{httpAddress}\" for farm \"{farmData.Name}\"");
}
}
return(null);
}
private async Task CreateFarmAsync(FarmData farmData)
{
// Require a minimum of a name
if (string.IsNullOrWhiteSpace(farmData.Name))
{
return;
}
Logger.LogInformation($"Creating farm \"{farmData.Name}\"");
var dbFarm = await FarmDataToFarmService.MakeFarmFromFarmDataAsync(farmData);
await FarmRepository.CreateAsync(dbFarm);
}
private string?ParseInstagram(FarmData farmData)
{
if (string.IsNullOrWhiteSpace(farmData.SocialMedia))
{
return(null);
}
var parts = farmData.SocialMedia.Split(' ', ',');
string?handle = parts.FirstOrDefault(p => p.Contains("@"));
if (string.IsNullOrWhiteSpace(handle) || !handle.StartsWith("@"))
{
return(null);
}
return(handle);
}
public async Task <Farm> MakeFarmFromFarmDataAsync(FarmData farmData)
{
PostcodeResult?postcodeLookup = await GetPostcodeResultAsync(farmData);
return(new Farm
{
Name = farmData.Name ?? "-- missing --",
Town = farmData.Town ?? postcodeLookup?.Parish,
County = farmData.County ?? postcodeLookup?.AdminCounty,
Postcode = postcodeLookup?.Postcode ?? farmData.Postcode,
Latitude = postcodeLookup?.Latitude,
Longitude = postcodeLookup?.Longitude,
GeoHash = ParseGeoHash(postcodeLookup),
Website = await ParseWebsiteAsync(farmData),
Instagram = ParseInstagram(farmData),
Facebook = await ParseFacebookAsync(farmData)
});
}
private async Task UpdateFarmAsync(FarmData farmData, Farm dbFarm, bool forceUpdate)
{
if (IsDifferent(farmData, dbFarm) || forceUpdate)
{
// Require a minimum of a name
if (string.IsNullOrWhiteSpace(farmData.Name))
{
return;
}
Logger.LogInformation($"Updating farm \"{farmData.Name}\"");
var updatedFarm = await FarmDataToFarmService.MakeFarmFromFarmDataAsync(farmData);
// Mark it as an update of the existing db record
updatedFarm.VersionNumber = dbFarm.VersionNumber;
await FarmRepository.UpdateAsync(updatedFarm);
}
}
private async Task <PostcodeResult?> GetPostcodeResultAsync(FarmData farmData)
{
PostcodeResult?postcodeLookup = null;
if (!string.IsNullOrWhiteSpace(farmData.Postcode))
{
postcodeLookup = await PostcodeService.GetPostcodeInfoAsync(farmData.Postcode);
}
if (postcodeLookup == null && !string.IsNullOrWhiteSpace(farmData.Town))
{
postcodeLookup = await PostcodeService.GetPlaceAsync(farmData.Town);
}
if (postcodeLookup == null && !string.IsNullOrWhiteSpace(farmData.County))
{
postcodeLookup = await PostcodeService.GetPlaceAsync(farmData.County);
}
return(postcodeLookup);
}