public static void Upload(string apiKey, string serviceName) { DateTime startDateTime = DateTime.Now; //Get all the provider data. List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload(); Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Get the complete list of insurances. startDateTime = DateTime.Now; List <string> insurances = new List <string>(); foreach (KyruusDataStructure p in providers) { if (p.insurance_accepted == null || p.insurance_accepted.Length == 0) { continue; } insurances.AddRange(p.insurance_accepted.Select(i => i.name)); } Console.WriteLine($"{insurances.Count} insurances. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //De-dupe the insurances. startDateTime = DateTime.Now; insurances = insurances .Where(i => string.IsNullOrWhiteSpace(i) == false) .Distinct() .ToList(); //Now assign the ID InsuranceIndexDataStructure[] insurancesIndexList = new InsuranceIndexDataStructure[insurances.Count]; for (int i = 0; i < insurances.Count; i++) { insurancesIndexList[i] = new InsuranceIndexDataStructure { id = (i + 1).ToString(), insurance = insurances[i] }; } Console.WriteLine($"{insurancesIndexList.Length} insurances to upload. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Drop and recreate the Azure Index. startDateTime = DateTime.Now; RecreateIndex(); Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Populate index. startDateTime = DateTime.Now; IndexLoader.MergeOrUpload(insurancesIndexList.ToList(), apiKey, serviceName, "insurances"); Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); }
public static void Upload(string apiKey, string serviceName) { DateTime startDateTime = DateTime.Now; //Get all the provider data. List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload(); Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Get the complete list of aliases and specialties. startDateTime = DateTime.Now; List <SpecialtyAliasAndType> specialtiesAliasesAndTypes = AccumulateAllSpecialitiesAliasesAndTypes(providers); Console.WriteLine($"{specialtiesAliasesAndTypes.Count} specialtiesAliasesAndTypes. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //De-dupe the list and remove specialties for where we have aliases. startDateTime = DateTime.Now; specialtiesAliasesAndTypes = DeDupe(specialtiesAliasesAndTypes); Console.WriteLine($"{specialtiesAliasesAndTypes.Count} de-duped. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); startDateTime = DateTime.Now; specialtiesAliasesAndTypes = RemoveSpecialtyEntriesWithAliases(specialtiesAliasesAndTypes); Console.WriteLine($"{specialtiesAliasesAndTypes.Count} entries after removing some specialty entries. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Drop and recreate the Azure Index. startDateTime = DateTime.Now; RecreateIndex(); Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Populate specialties index. startDateTime = DateTime.Now; SpecialtyIndexDataStructure[] specialties = new SpecialtyIndexDataStructure[specialtiesAliasesAndTypes.Count]; for (int i = 0; i < specialtiesAliasesAndTypes.Count; i++) { specialties[i] = new SpecialtyIndexDataStructure { alias = specialtiesAliasesAndTypes[i].Alias, id = (i + 1).ToString(), specialty = specialtiesAliasesAndTypes[i].Specialty }; } IndexLoader.MergeOrUpload(specialties.ToList(), apiKey, serviceName, "specialties"); Console.WriteLine($"Upload index response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); }
public static void Upload(string apiKey, string serviceName) { //Get all the provider data. DateTime startDateTime = DateTime.Now; List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload(); Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); startDateTime = DateTime.Now; List <Location> locations = new List <Location>(); foreach (KyruusDataStructure p in providers) { foreach (Common.Location l in p.locations) { if (l.coordinates == null) { continue; } locations.Add(new Location { coordinates = GeographyPoint.Create(l.coordinates.lat, l.coordinates.lon), id = l.id //Kyruus has multiple 'id's for the same external_id. But if we use the external_id and that changes (planned to switch over to Cerner's ID), //we have to rebuild both locations and providers indexes and coordinate such with Sitecore releases. Can't use coordinates either to ID a facility //since that can change slightly too. Keep in mind that we need to attach the IDs to the providers too... }); } } Console.WriteLine($"{providers.Count} providers. {locations.Count} locations. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Get unique list of locations. startDateTime = DateTime.Now; locations = locations.Distinct(new LocationComparer()).ToList(); Console.WriteLine($"{locations.Count} unique locations. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Upload the data to Azure Search startDateTime = DateTime.Now; IndexLoader.MergeOrUpload(locations, apiKey, serviceName, "locations"); Console.WriteLine($"Index uploaded. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); }
public static void Upload(string apiKey, string serviceName) { DateTime startDateTime = DateTime.Now; //Get all the provider data. List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload(); Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Get the complete list of primary care and non-primary care conditions. startDateTime = DateTime.Now; List <string> primaryCareConditions; List <string> nonPrimaryCareConditions; GetConditions(providers, out primaryCareConditions, out nonPrimaryCareConditions); Console.WriteLine($"{primaryCareConditions.Count} primaryCareConditions. {nonPrimaryCareConditions.Count} nonPrimaryCareConditions. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //De-dupe the primary and non primary care conditions. startDateTime = DateTime.Now; primaryCareConditions = primaryCareConditions .Where(p => string.IsNullOrWhiteSpace(p) == false) .Distinct() .ToList(); nonPrimaryCareConditions = nonPrimaryCareConditions .Where(n => string.IsNullOrWhiteSpace(n) == false) .Distinct() .ToList(); //Put the two condition types together. List <ConditionIndexDataStructure> conditions = primaryCareConditions .Select(p => new ConditionIndexDataStructure { condition = p, id = string.Empty, isPrimaryCare = true }) .ToList(); conditions.AddRange(nonPrimaryCareConditions .Select(n => new ConditionIndexDataStructure { condition = n, id = string.Empty, isPrimaryCare = false }) .OrderBy(c => c.condition) .ThenByDescending(c => c.isPrimaryCare) .ToList()); //Note that we will load this index in a reasonable order so as to avoid index fragmentation and speed up inquiry times. //Now assign the ID for (int c = 0; c < conditions.Count; c++) { conditions[c].id = (c + 1).ToString(); } Console.WriteLine($"{primaryCareConditions.Count} unique primaryCareConditions. {nonPrimaryCareConditions.Count} unique nonPrimaryCareConditions. {conditions.Count} combined conditions. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Drop and recreate the Azure Index. startDateTime = DateTime.Now; RecreateIndex(); Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Populate conditions index. startDateTime = DateTime.Now; IndexLoader.MergeOrUpload(conditions, apiKey, serviceName, "conditions"); Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); }
public static void Upload(string apiKey, string serviceName) { //Get all the provider data. DateTime startDateTime = DateTime.Now; List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload(); Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); startDateTime = DateTime.Now; List <string> highRankingNetworks = new List <string>() { "bann", "bep", "bhwr", "bumg" }; //TODO Need to get this from app settings. Random random = new Random(DateTime.Now.Millisecond); List <ProviderIndex> providerIndexList = new List <ProviderIndex>(); foreach (KyruusDataStructure p in providers) { #region Assuming properties can be null. This is what the code here is about -- expecting the worst. //Accepted insurances List <string> acceptedInsurances = new List <string>(); if (p.insurance_accepted != null) { acceptedInsurances = p.insurance_accepted .Where(i => string.IsNullOrWhiteSpace(i.name) == false) .Select(i => i.name) .ToList(); } //Ages seen List <string> agesSeen = new List <string>(); if (p.age_groups_seen != null) { agesSeen = p.age_groups_seen .Where(a => string.IsNullOrWhiteSpace(a.name) == false) .Select(a => a.name) .ToList(); } //Cities List <string> cities = new List <string>(); if (p.locations != null) { cities = p.locations .Where(l => string.IsNullOrWhiteSpace(l.city) == false) .Select(l => l.city) .ToList(); } //Conditions List <string> conditions = new List <string>(); if (p.scope_of_practice != null && p.scope_of_practice.concepts != null) { conditions = p.scope_of_practice.concepts .Where(c => c.searchability == "searchable" && string.IsNullOrWhiteSpace(c.name) == false) .Select(c => c.name) .ToList(); } //Languages List <string> languages = new List <string>(); if (p.languages != null) { languages = p.languages .Where(l => string.IsNullOrWhiteSpace(l.language) == false) .Select(l => l.language) .ToList(); } //Location IDs List <string> locationIds = new List <string>(); if (p.locations != null) { locationIds = p.locations .Where(l => string.IsNullOrWhiteSpace(l.id) == false) .Select(l => l.id) .ToList(); } //Names //p.name.full_name does not have the p.preferred_name in the first name part. string firstName = string.Empty; string lastName = string.Empty; string firstAndLastName = string.Empty; if (p.name != null) { if (string.IsNullOrWhiteSpace(p.preferred_name)) { firstName = p.name.first_name; } else { firstName = p.preferred_name; } if (string.IsNullOrWhiteSpace(firstName)) { firstName = string.Empty; } lastName = p.name.last_name; if (string.IsNullOrWhiteSpace(lastName)) { lastName = string.Empty; } } if (firstName.Length == 0) { firstAndLastName = lastName; } else { firstAndLastName = firstName + " " + lastName; } //Network affiliations List <string> networkAffiliations = new List <string>(); if (p.network_affiliations != null) { networkAffiliations = p.network_affiliations .Where(n => n.type == "Hospital" && string.IsNullOrWhiteSpace(n.name) == false) .Select(n => n.name) .ToList(); } //Specialties List <string> specialties = new List <string>(); if (p.specialties != null) { //Add specialties specialties = p.specialties .Select(s => s.specialty) .ToList(); //Add subspecialties specialties.AddRange( p.specialties .Select(s => s.subspecialty) .ToList()); //Add aliases specialties.AddRange( p.specialties .SelectMany(s => s.aliases) .Select(a => a.name) .ToList()); //Remove any null entries specialties = specialties .Where(s => string.IsNullOrWhiteSpace(s) == false) .ToList(); //Distinct the list specialties = specialties .Distinct() .ToList(); } int searchRank = 100; if (p.networks != null) { if (highRankingNetworks .Intersect(p.networks.Select(n => n.network.ToLower()) .ToList()) .Count() > 0) { searchRank = 0; } } //Zip codes List <string> zipCodes = new List <string>(); if (p.locations != null) { zipCodes = p.locations .Where(l => string.IsNullOrWhiteSpace(l.zip) == false) .Select(l => l.zip) .ToList(); } #endregion ProviderIndex pi = new ProviderIndex { acceptedInsurances = acceptedInsurances, acceptedInsurancesLower = acceptedInsurances .Select(i => i.ToLower()) .ToList(), acceptNewPatients = p.accepting_new_patients, agesSeen = agesSeen, cities = cities, conditions = conditions, conditionsLower = conditions.Select(c => c.ToLower()).ToList(), //TODO recheck when you need lower. firstAndLastName = firstAndLastName, firstAndLastNameLower = firstAndLastName.ToLower(), id = p.id.ToString(), isMale = p.gender == "Male", isPrimaryCare = p.is_primary_care, languages = languages, locationIds = locationIds, networkAffiliations = networkAffiliations, providerType = p.provider_type, randomNumber = random.Next(0, int.MaxValue), //Use a huge random range to minimize chances of getting the same number again later on. searchRank = searchRank, specialties = specialties, specialtiesLower = specialties.Select(s => s.ToLower()).ToList(), zipCodes = zipCodes }; providerIndexList.Add(pi); } Console.WriteLine($"{providers.Count} providers in memory. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Upload the data to Azure Search startDateTime = DateTime.Now; IndexLoader.MergeOrUpload(providerIndexList, apiKey, serviceName, "providers", 500); Console.WriteLine($"Index uploaded. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); }
public static void Upload(string apiKey, string serviceName) { DateTime startDateTime = DateTime.Now; //Get all the provider data. List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload(); Console.WriteLine($"{providers.Count} providers fetched. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Get the complete list of names. startDateTime = DateTime.Now; List <string> firstAndLastNames = new List <string>(); foreach (KyruusDataStructure p in providers) { if (p.name == null) { continue; } string firstName = string.Empty; string lastName = string.Empty; string firstAndLastName = string.Empty; //p.name.full_name does not have the p.preferred_name in the first name part. if (string.IsNullOrWhiteSpace(p.preferred_name)) { firstName = p.name.first_name; } else { firstName = p.preferred_name; } if (string.IsNullOrWhiteSpace(firstName)) { firstName = string.Empty; } lastName = p.name.last_name; if (string.IsNullOrWhiteSpace(lastName)) { lastName = string.Empty; } if (firstName.Length == 0) { firstAndLastName = lastName; } else { firstAndLastName = firstName + " " + lastName; } firstAndLastNames.Add(firstAndLastName); } Console.WriteLine($"{firstAndLastNames.Count} names. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //De-dupe the names. startDateTime = DateTime.Now; firstAndLastNames = firstAndLastNames .Where(p => string.IsNullOrWhiteSpace(p) == false) .Distinct() .ToList(); //Now assign the ID NameIndexDataStructure[] names = new NameIndexDataStructure[firstAndLastNames.Count]; for (int n = 0; n < firstAndLastNames.Count; n++) { names[n] = new NameIndexDataStructure { firstAndLastName = firstAndLastNames[n], id = (n + 1).ToString() }; } Console.WriteLine($"{names.Length} names to be uploaded. Response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Drop and recreate the Azure Index. startDateTime = DateTime.Now; RecreateIndex(); Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); //Populate index. startDateTime = DateTime.Now; IndexLoader.MergeOrUpload(names.ToList(), apiKey, serviceName, "names"); Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}"); }