public static void Upload(string apiKey, string serviceName)
{
DateTime startDateTime = DateTime.Now;
//Get all the provider data.
List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();
Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Get the complete list of insurances.
startDateTime = DateTime.Now;
List <string> insurances = new List <string>();
foreach (KyruusDataStructure p in providers)
{
if (p.insurance_accepted == null || p.insurance_accepted.Length == 0)
{
continue;
}
insurances.AddRange(p.insurance_accepted.Select(i => i.name));
}
Console.WriteLine($"{insurances.Count} insurances. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//De-dupe the insurances.
startDateTime = DateTime.Now;
insurances = insurances
.Where(i => string.IsNullOrWhiteSpace(i) == false)
.Distinct()
.ToList();
//Now assign the ID
InsuranceIndexDataStructure[] insurancesIndexList = new InsuranceIndexDataStructure[insurances.Count];
for (int i = 0; i < insurances.Count; i++)
{
insurancesIndexList[i] = new InsuranceIndexDataStructure {
id = (i + 1).ToString(), insurance = insurances[i]
};
}
Console.WriteLine($"{insurancesIndexList.Length} insurances to upload. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Drop and recreate the Azure Index.
startDateTime = DateTime.Now;
RecreateIndex();
Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Populate index.
startDateTime = DateTime.Now;
IndexLoader.MergeOrUpload(insurancesIndexList.ToList(), apiKey, serviceName, "insurances");
Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
}
public static void Upload(string apiKey, string serviceName)
{
DateTime startDateTime = DateTime.Now;
//Get all the provider data.
List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();
Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Get the complete list of aliases and specialties.
startDateTime = DateTime.Now;
List <SpecialtyAliasAndType> specialtiesAliasesAndTypes = AccumulateAllSpecialitiesAliasesAndTypes(providers);
Console.WriteLine($"{specialtiesAliasesAndTypes.Count} specialtiesAliasesAndTypes. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//De-dupe the list and remove specialties for where we have aliases.
startDateTime = DateTime.Now;
specialtiesAliasesAndTypes = DeDupe(specialtiesAliasesAndTypes);
Console.WriteLine($"{specialtiesAliasesAndTypes.Count} de-duped. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
startDateTime = DateTime.Now;
specialtiesAliasesAndTypes = RemoveSpecialtyEntriesWithAliases(specialtiesAliasesAndTypes);
Console.WriteLine($"{specialtiesAliasesAndTypes.Count} entries after removing some specialty entries. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Drop and recreate the Azure Index.
startDateTime = DateTime.Now;
RecreateIndex();
Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Populate specialties index.
startDateTime = DateTime.Now;
SpecialtyIndexDataStructure[] specialties = new SpecialtyIndexDataStructure[specialtiesAliasesAndTypes.Count];
for (int i = 0; i < specialtiesAliasesAndTypes.Count; i++)
{
specialties[i] = new SpecialtyIndexDataStructure
{
alias = specialtiesAliasesAndTypes[i].Alias,
id = (i + 1).ToString(),
specialty = specialtiesAliasesAndTypes[i].Specialty
};
}
IndexLoader.MergeOrUpload(specialties.ToList(), apiKey, serviceName, "specialties");
Console.WriteLine($"Upload index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
}
public static void Upload(string apiKey, string serviceName)
{
//Get all the provider data.
DateTime startDateTime = DateTime.Now;
List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();
Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
startDateTime = DateTime.Now;
List <Location> locations = new List <Location>();
foreach (KyruusDataStructure p in providers)
{
foreach (Common.Location l in p.locations)
{
if (l.coordinates == null)
{
continue;
}
locations.Add(new Location
{
coordinates = GeographyPoint.Create(l.coordinates.lat, l.coordinates.lon),
id = l.id //Kyruus has multiple 'id's for the same external_id. But if we use the external_id and that changes (planned to switch over to Cerner's ID),
//we have to rebuild both locations and providers indexes and coordinate such with Sitecore releases. Can't use coordinates either to ID a facility
//since that can change slightly too. Keep in mind that we need to attach the IDs to the providers too...
});
}
}
Console.WriteLine($"{providers.Count} providers. {locations.Count} locations. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Get unique list of locations.
startDateTime = DateTime.Now;
locations = locations.Distinct(new LocationComparer()).ToList();
Console.WriteLine($"{locations.Count} unique locations. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Upload the data to Azure Search
startDateTime = DateTime.Now;
IndexLoader.MergeOrUpload(locations, apiKey, serviceName, "locations");
Console.WriteLine($"Index uploaded. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
}
public static void Upload(string apiKey, string serviceName)
{
DateTime startDateTime = DateTime.Now;
//Get all the provider data.
List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();
Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Get the complete list of primary care and non-primary care conditions.
startDateTime = DateTime.Now;
List <string> primaryCareConditions;
List <string> nonPrimaryCareConditions;
GetConditions(providers, out primaryCareConditions, out nonPrimaryCareConditions);
Console.WriteLine($"{primaryCareConditions.Count} primaryCareConditions. {nonPrimaryCareConditions.Count} nonPrimaryCareConditions. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//De-dupe the primary and non primary care conditions.
startDateTime = DateTime.Now;
primaryCareConditions = primaryCareConditions
.Where(p => string.IsNullOrWhiteSpace(p) == false)
.Distinct()
.ToList();
nonPrimaryCareConditions = nonPrimaryCareConditions
.Where(n => string.IsNullOrWhiteSpace(n) == false)
.Distinct()
.ToList();
//Put the two condition types together.
List <ConditionIndexDataStructure> conditions = primaryCareConditions
.Select(p => new ConditionIndexDataStructure
{
condition = p,
id = string.Empty,
isPrimaryCare = true
})
.ToList();
conditions.AddRange(nonPrimaryCareConditions
.Select(n => new ConditionIndexDataStructure
{
condition = n,
id = string.Empty,
isPrimaryCare = false
})
.OrderBy(c => c.condition)
.ThenByDescending(c => c.isPrimaryCare)
.ToList());
//Note that we will load this index in a reasonable order so as to avoid index fragmentation and speed up inquiry times.
//Now assign the ID
for (int c = 0; c < conditions.Count; c++)
{
conditions[c].id = (c + 1).ToString();
}
Console.WriteLine($"{primaryCareConditions.Count} unique primaryCareConditions. {nonPrimaryCareConditions.Count} unique nonPrimaryCareConditions. {conditions.Count} combined conditions. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Drop and recreate the Azure Index.
startDateTime = DateTime.Now;
RecreateIndex();
Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Populate conditions index.
startDateTime = DateTime.Now;
IndexLoader.MergeOrUpload(conditions, apiKey, serviceName, "conditions");
Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
}
public static void Upload(string apiKey, string serviceName)
{
//Get all the provider data.
DateTime startDateTime = DateTime.Now;
List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();
Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
startDateTime = DateTime.Now;
List <string> highRankingNetworks = new List <string>()
{
"bann", "bep", "bhwr", "bumg"
}; //TODO Need to get this from app settings.
Random random = new Random(DateTime.Now.Millisecond);
List <ProviderIndex> providerIndexList = new List <ProviderIndex>();
foreach (KyruusDataStructure p in providers)
{
#region Assuming properties can be null. This is what the code here is about -- expecting the worst.
//Accepted insurances
List <string> acceptedInsurances = new List <string>();
if (p.insurance_accepted != null)
{
acceptedInsurances = p.insurance_accepted
.Where(i => string.IsNullOrWhiteSpace(i.name) == false)
.Select(i => i.name)
.ToList();
}
//Ages seen
List <string> agesSeen = new List <string>();
if (p.age_groups_seen != null)
{
agesSeen = p.age_groups_seen
.Where(a => string.IsNullOrWhiteSpace(a.name) == false)
.Select(a => a.name)
.ToList();
}
//Cities
List <string> cities = new List <string>();
if (p.locations != null)
{
cities = p.locations
.Where(l => string.IsNullOrWhiteSpace(l.city) == false)
.Select(l => l.city)
.ToList();
}
//Conditions
List <string> conditions = new List <string>();
if (p.scope_of_practice != null && p.scope_of_practice.concepts != null)
{
conditions = p.scope_of_practice.concepts
.Where(c => c.searchability == "searchable" && string.IsNullOrWhiteSpace(c.name) == false)
.Select(c => c.name)
.ToList();
}
//Languages
List <string> languages = new List <string>();
if (p.languages != null)
{
languages = p.languages
.Where(l => string.IsNullOrWhiteSpace(l.language) == false)
.Select(l => l.language)
.ToList();
}
//Location IDs
List <string> locationIds = new List <string>();
if (p.locations != null)
{
locationIds = p.locations
.Where(l => string.IsNullOrWhiteSpace(l.id) == false)
.Select(l => l.id)
.ToList();
}
//Names
//p.name.full_name does not have the p.preferred_name in the first name part.
string firstName = string.Empty;
string lastName = string.Empty;
string firstAndLastName = string.Empty;
if (p.name != null)
{
if (string.IsNullOrWhiteSpace(p.preferred_name))
{
firstName = p.name.first_name;
}
else
{
firstName = p.preferred_name;
}
if (string.IsNullOrWhiteSpace(firstName))
{
firstName = string.Empty;
}
lastName = p.name.last_name;
if (string.IsNullOrWhiteSpace(lastName))
{
lastName = string.Empty;
}
}
if (firstName.Length == 0)
{
firstAndLastName = lastName;
}
else
{
firstAndLastName = firstName + " " + lastName;
}
//Network affiliations
List <string> networkAffiliations = new List <string>();
if (p.network_affiliations != null)
{
networkAffiliations = p.network_affiliations
.Where(n => n.type == "Hospital" && string.IsNullOrWhiteSpace(n.name) == false)
.Select(n => n.name)
.ToList();
}
//Specialties
List <string> specialties = new List <string>();
if (p.specialties != null)
{
//Add specialties
specialties = p.specialties
.Select(s => s.specialty)
.ToList();
//Add subspecialties
specialties.AddRange(
p.specialties
.Select(s => s.subspecialty)
.ToList());
//Add aliases
specialties.AddRange(
p.specialties
.SelectMany(s => s.aliases)
.Select(a => a.name)
.ToList());
//Remove any null entries
specialties = specialties
.Where(s => string.IsNullOrWhiteSpace(s) == false)
.ToList();
//Distinct the list
specialties = specialties
.Distinct()
.ToList();
}
int searchRank = 100;
if (p.networks != null)
{
if (highRankingNetworks
.Intersect(p.networks.Select(n => n.network.ToLower())
.ToList())
.Count() > 0)
{
searchRank = 0;
}
}
//Zip codes
List <string> zipCodes = new List <string>();
if (p.locations != null)
{
zipCodes = p.locations
.Where(l => string.IsNullOrWhiteSpace(l.zip) == false)
.Select(l => l.zip)
.ToList();
}
#endregion
ProviderIndex pi = new ProviderIndex
{
acceptedInsurances = acceptedInsurances,
acceptedInsurancesLower = acceptedInsurances
.Select(i => i.ToLower())
.ToList(),
acceptNewPatients = p.accepting_new_patients,
agesSeen = agesSeen,
cities = cities,
conditions = conditions,
conditionsLower = conditions.Select(c => c.ToLower()).ToList(), //TODO recheck when you need lower.
firstAndLastName = firstAndLastName,
firstAndLastNameLower = firstAndLastName.ToLower(),
id = p.id.ToString(),
isMale = p.gender == "Male",
isPrimaryCare = p.is_primary_care,
languages = languages,
locationIds = locationIds,
networkAffiliations = networkAffiliations,
providerType = p.provider_type,
randomNumber = random.Next(0, int.MaxValue), //Use a huge random range to minimize chances of getting the same number again later on.
searchRank = searchRank,
specialties = specialties,
specialtiesLower = specialties.Select(s => s.ToLower()).ToList(),
zipCodes = zipCodes
};
providerIndexList.Add(pi);
}
Console.WriteLine($"{providers.Count} providers in memory. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Upload the data to Azure Search
startDateTime = DateTime.Now;
IndexLoader.MergeOrUpload(providerIndexList, apiKey, serviceName, "providers", 500);
Console.WriteLine($"Index uploaded. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
}
public static void Upload(string apiKey, string serviceName)
{
DateTime startDateTime = DateTime.Now;
//Get all the provider data.
List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();
Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Get the complete list of names.
startDateTime = DateTime.Now;
List <string> firstAndLastNames = new List <string>();
foreach (KyruusDataStructure p in providers)
{
if (p.name == null)
{
continue;
}
string firstName = string.Empty;
string lastName = string.Empty;
string firstAndLastName = string.Empty;
//p.name.full_name does not have the p.preferred_name in the first name part.
if (string.IsNullOrWhiteSpace(p.preferred_name))
{
firstName = p.name.first_name;
}
else
{
firstName = p.preferred_name;
}
if (string.IsNullOrWhiteSpace(firstName))
{
firstName = string.Empty;
}
lastName = p.name.last_name;
if (string.IsNullOrWhiteSpace(lastName))
{
lastName = string.Empty;
}
if (firstName.Length == 0)
{
firstAndLastName = lastName;
}
else
{
firstAndLastName = firstName + " " + lastName;
}
firstAndLastNames.Add(firstAndLastName);
}
Console.WriteLine($"{firstAndLastNames.Count} names. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//De-dupe the names.
startDateTime = DateTime.Now;
firstAndLastNames = firstAndLastNames
.Where(p => string.IsNullOrWhiteSpace(p) == false)
.Distinct()
.ToList();
//Now assign the ID
NameIndexDataStructure[] names = new NameIndexDataStructure[firstAndLastNames.Count];
for (int n = 0; n < firstAndLastNames.Count; n++)
{
names[n] = new NameIndexDataStructure {
firstAndLastName = firstAndLastNames[n], id = (n + 1).ToString()
};
}
Console.WriteLine($"{names.Length} names to be uploaded. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Drop and recreate the Azure Index.
startDateTime = DateTime.Now;
RecreateIndex();
Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
//Populate index.
startDateTime = DateTime.Now;
IndexLoader.MergeOrUpload(names.ToList(), apiKey, serviceName, "names");
Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
}