Esempio n. 1
0
        public static void Upload(string apiKey, string serviceName)
        {
            DateTime startDateTime = DateTime.Now;

            //Get all the provider data.
            List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();

            Console.WriteLine($"{providers.Count} providers fetched.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Get the complete list of insurances.
            startDateTime = DateTime.Now;
            List <string> insurances = new List <string>();

            foreach (KyruusDataStructure p in providers)
            {
                if (p.insurance_accepted == null || p.insurance_accepted.Length == 0)
                {
                    continue;
                }
                insurances.AddRange(p.insurance_accepted.Select(i => i.name));
            }
            Console.WriteLine($"{insurances.Count} insurances.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //De-dupe the insurances.
            startDateTime = DateTime.Now;
            insurances    = insurances
                            .Where(i => string.IsNullOrWhiteSpace(i) == false)
                            .Distinct()
                            .ToList();
            //Now assign the ID
            InsuranceIndexDataStructure[] insurancesIndexList = new InsuranceIndexDataStructure[insurances.Count];
            for (int i = 0; i < insurances.Count; i++)
            {
                insurancesIndexList[i] = new InsuranceIndexDataStructure {
                    id = (i + 1).ToString(), insurance = insurances[i]
                };
            }
            Console.WriteLine($"{insurancesIndexList.Length} insurances to upload.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Drop and recreate the Azure Index.
            startDateTime = DateTime.Now;
            RecreateIndex();
            Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Populate index.
            startDateTime = DateTime.Now;
            IndexLoader.MergeOrUpload(insurancesIndexList.ToList(), apiKey, serviceName, "insurances");
            Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
        }
Esempio n. 2
0
        public static void Upload(string apiKey, string serviceName)
        {
            DateTime startDateTime = DateTime.Now;

            //Get all the provider data.
            List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();

            Console.WriteLine($"{providers.Count} providers fetched.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Get the complete list of aliases and specialties.
            startDateTime = DateTime.Now;
            List <SpecialtyAliasAndType> specialtiesAliasesAndTypes = AccumulateAllSpecialitiesAliasesAndTypes(providers);

            Console.WriteLine($"{specialtiesAliasesAndTypes.Count} specialtiesAliasesAndTypes.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //De-dupe the list and remove specialties for where we have aliases.
            startDateTime = DateTime.Now;
            specialtiesAliasesAndTypes = DeDupe(specialtiesAliasesAndTypes);
            Console.WriteLine($"{specialtiesAliasesAndTypes.Count} de-duped.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            startDateTime = DateTime.Now;
            specialtiesAliasesAndTypes = RemoveSpecialtyEntriesWithAliases(specialtiesAliasesAndTypes);
            Console.WriteLine($"{specialtiesAliasesAndTypes.Count} entries after removing some specialty entries.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Drop and recreate the Azure Index.
            startDateTime = DateTime.Now;
            RecreateIndex();
            Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Populate specialties index.
            startDateTime = DateTime.Now;
            SpecialtyIndexDataStructure[] specialties = new SpecialtyIndexDataStructure[specialtiesAliasesAndTypes.Count];
            for (int i = 0; i < specialtiesAliasesAndTypes.Count; i++)
            {
                specialties[i] = new SpecialtyIndexDataStructure
                {
                    alias     = specialtiesAliasesAndTypes[i].Alias,
                    id        = (i + 1).ToString(),
                    specialty = specialtiesAliasesAndTypes[i].Specialty
                };
            }
            IndexLoader.MergeOrUpload(specialties.ToList(), apiKey, serviceName, "specialties");
            Console.WriteLine($"Upload index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
        }
Esempio n. 3
0
        public static void Upload(string apiKey, string serviceName)
        {
            //Get all the provider data.
            DateTime startDateTime = DateTime.Now;
            List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();

            Console.WriteLine($"{providers.Count} providers fetched.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            startDateTime = DateTime.Now;
            List <Location> locations = new List <Location>();

            foreach (KyruusDataStructure p in providers)
            {
                foreach (Common.Location l in p.locations)
                {
                    if (l.coordinates == null)
                    {
                        continue;
                    }
                    locations.Add(new Location
                    {
                        coordinates = GeographyPoint.Create(l.coordinates.lat, l.coordinates.lon),
                        id          = l.id //Kyruus has multiple 'id's for the same external_id.  But if we use the external_id and that changes (planned to switch over to Cerner's ID),
                                           //we have to rebuild both locations and providers indexes and coordinate such with Sitecore releases.  Can't use coordinates either to ID a facility
                                           //since that can change slightly too.  Keep in mind that we need to attach the IDs to the providers too...
                    });
                }
            }
            Console.WriteLine($"{providers.Count} providers.  {locations.Count} locations.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Get unique list of locations.
            startDateTime = DateTime.Now;
            locations     = locations.Distinct(new LocationComparer()).ToList();
            Console.WriteLine($"{locations.Count} unique locations.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Upload the data to Azure Search
            startDateTime = DateTime.Now;
            IndexLoader.MergeOrUpload(locations, apiKey, serviceName, "locations");
            Console.WriteLine($"Index uploaded.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
        }
Esempio n. 4
0
        public static void Upload(string apiKey, string serviceName)
        {
            DateTime startDateTime = DateTime.Now;

            //Get all the provider data.
            List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();

            Console.WriteLine($"{providers.Count} providers fetched.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Get the complete list of primary care and non-primary care conditions.
            startDateTime = DateTime.Now;
            List <string> primaryCareConditions;
            List <string> nonPrimaryCareConditions;

            GetConditions(providers, out primaryCareConditions, out nonPrimaryCareConditions);
            Console.WriteLine($"{primaryCareConditions.Count} primaryCareConditions.  {nonPrimaryCareConditions.Count} nonPrimaryCareConditions.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //De-dupe the primary and non primary care conditions.
            startDateTime         = DateTime.Now;
            primaryCareConditions = primaryCareConditions
                                    .Where(p => string.IsNullOrWhiteSpace(p) == false)
                                    .Distinct()
                                    .ToList();
            nonPrimaryCareConditions = nonPrimaryCareConditions
                                       .Where(n => string.IsNullOrWhiteSpace(n) == false)
                                       .Distinct()
                                       .ToList();
            //Put the two condition types together.
            List <ConditionIndexDataStructure> conditions = primaryCareConditions
                                                            .Select(p => new ConditionIndexDataStructure
            {
                condition     = p,
                id            = string.Empty,
                isPrimaryCare = true
            })
                                                            .ToList();

            conditions.AddRange(nonPrimaryCareConditions
                                .Select(n => new ConditionIndexDataStructure
            {
                condition     = n,
                id            = string.Empty,
                isPrimaryCare = false
            })
                                .OrderBy(c => c.condition)
                                .ThenByDescending(c => c.isPrimaryCare)
                                .ToList());
            //Note that we will load this index in a reasonable order so as to avoid index fragmentation and speed up inquiry times.
            //Now assign the ID
            for (int c = 0; c < conditions.Count; c++)
            {
                conditions[c].id = (c + 1).ToString();
            }
            Console.WriteLine($"{primaryCareConditions.Count} unique primaryCareConditions.  {nonPrimaryCareConditions.Count} unique nonPrimaryCareConditions.  {conditions.Count} combined conditions.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Drop and recreate the Azure Index.
            startDateTime = DateTime.Now;
            RecreateIndex();
            Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Populate conditions index.
            startDateTime = DateTime.Now;
            IndexLoader.MergeOrUpload(conditions, apiKey, serviceName, "conditions");
            Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
        }
Esempio n. 5
0
        public static void Upload(string apiKey, string serviceName)
        {
            //Get all the provider data.
            DateTime startDateTime = DateTime.Now;
            List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();

            Console.WriteLine($"{providers.Count} providers fetched.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            startDateTime = DateTime.Now;
            List <string> highRankingNetworks = new List <string>()
            {
                "bann", "bep", "bhwr", "bumg"
            };                                                                                          //TODO Need to get this from app settings.

            Random random = new Random(DateTime.Now.Millisecond);

            List <ProviderIndex> providerIndexList = new List <ProviderIndex>();

            foreach (KyruusDataStructure p in providers)
            {
                #region Assuming properties can be null.  This is what the code here is about -- expecting the worst.
                //Accepted insurances
                List <string> acceptedInsurances = new List <string>();
                if (p.insurance_accepted != null)
                {
                    acceptedInsurances = p.insurance_accepted
                                         .Where(i => string.IsNullOrWhiteSpace(i.name) == false)
                                         .Select(i => i.name)
                                         .ToList();
                }
                //Ages seen
                List <string> agesSeen = new List <string>();
                if (p.age_groups_seen != null)
                {
                    agesSeen = p.age_groups_seen
                               .Where(a => string.IsNullOrWhiteSpace(a.name) == false)
                               .Select(a => a.name)
                               .ToList();
                }
                //Cities
                List <string> cities = new List <string>();
                if (p.locations != null)
                {
                    cities = p.locations
                             .Where(l => string.IsNullOrWhiteSpace(l.city) == false)
                             .Select(l => l.city)
                             .ToList();
                }
                //Conditions
                List <string> conditions = new List <string>();
                if (p.scope_of_practice != null && p.scope_of_practice.concepts != null)
                {
                    conditions = p.scope_of_practice.concepts
                                 .Where(c => c.searchability == "searchable" && string.IsNullOrWhiteSpace(c.name) == false)
                                 .Select(c => c.name)
                                 .ToList();
                }
                //Languages
                List <string> languages = new List <string>();
                if (p.languages != null)
                {
                    languages = p.languages
                                .Where(l => string.IsNullOrWhiteSpace(l.language) == false)
                                .Select(l => l.language)
                                .ToList();
                }
                //Location IDs
                List <string> locationIds = new List <string>();
                if (p.locations != null)
                {
                    locationIds = p.locations
                                  .Where(l => string.IsNullOrWhiteSpace(l.id) == false)
                                  .Select(l => l.id)
                                  .ToList();
                }
                //Names
                //p.name.full_name does not have the p.preferred_name in the first name part.
                string firstName        = string.Empty;
                string lastName         = string.Empty;
                string firstAndLastName = string.Empty;
                if (p.name != null)
                {
                    if (string.IsNullOrWhiteSpace(p.preferred_name))
                    {
                        firstName = p.name.first_name;
                    }
                    else
                    {
                        firstName = p.preferred_name;
                    }
                    if (string.IsNullOrWhiteSpace(firstName))
                    {
                        firstName = string.Empty;
                    }
                    lastName = p.name.last_name;
                    if (string.IsNullOrWhiteSpace(lastName))
                    {
                        lastName = string.Empty;
                    }
                }
                if (firstName.Length == 0)
                {
                    firstAndLastName = lastName;
                }
                else
                {
                    firstAndLastName = firstName + " " + lastName;
                }
                //Network affiliations
                List <string> networkAffiliations = new List <string>();
                if (p.network_affiliations != null)
                {
                    networkAffiliations = p.network_affiliations
                                          .Where(n => n.type == "Hospital" && string.IsNullOrWhiteSpace(n.name) == false)
                                          .Select(n => n.name)
                                          .ToList();
                }
                //Specialties
                List <string> specialties = new List <string>();
                if (p.specialties != null)
                {
                    //Add specialties
                    specialties = p.specialties
                                  .Select(s => s.specialty)
                                  .ToList();
                    //Add subspecialties
                    specialties.AddRange(
                        p.specialties
                        .Select(s => s.subspecialty)
                        .ToList());
                    //Add aliases
                    specialties.AddRange(
                        p.specialties
                        .SelectMany(s => s.aliases)
                        .Select(a => a.name)
                        .ToList());
                    //Remove any null entries
                    specialties = specialties
                                  .Where(s => string.IsNullOrWhiteSpace(s) == false)
                                  .ToList();
                    //Distinct the list
                    specialties = specialties
                                  .Distinct()
                                  .ToList();
                }
                int searchRank = 100;
                if (p.networks != null)
                {
                    if (highRankingNetworks
                        .Intersect(p.networks.Select(n => n.network.ToLower())
                                   .ToList())
                        .Count() > 0)
                    {
                        searchRank = 0;
                    }
                }
                //Zip codes
                List <string> zipCodes = new List <string>();
                if (p.locations != null)
                {
                    zipCodes = p.locations
                               .Where(l => string.IsNullOrWhiteSpace(l.zip) == false)
                               .Select(l => l.zip)
                               .ToList();
                }
                #endregion

                ProviderIndex pi = new ProviderIndex
                {
                    acceptedInsurances      = acceptedInsurances,
                    acceptedInsurancesLower = acceptedInsurances
                                              .Select(i => i.ToLower())
                                              .ToList(),
                    acceptNewPatients     = p.accepting_new_patients,
                    agesSeen              = agesSeen,
                    cities                = cities,
                    conditions            = conditions,
                    conditionsLower       = conditions.Select(c => c.ToLower()).ToList(), //TODO recheck when you need lower.
                    firstAndLastName      = firstAndLastName,
                    firstAndLastNameLower = firstAndLastName.ToLower(),
                    id                  = p.id.ToString(),
                    isMale              = p.gender == "Male",
                    isPrimaryCare       = p.is_primary_care,
                    languages           = languages,
                    locationIds         = locationIds,
                    networkAffiliations = networkAffiliations,
                    providerType        = p.provider_type,
                    randomNumber        = random.Next(0, int.MaxValue), //Use a huge random range to minimize chances of getting the same number again later on.
                    searchRank          = searchRank,
                    specialties         = specialties,
                    specialtiesLower    = specialties.Select(s => s.ToLower()).ToList(),
                    zipCodes            = zipCodes
                };
                providerIndexList.Add(pi);
            }
            Console.WriteLine($"{providers.Count} providers in memory.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Upload the data to Azure Search
            startDateTime = DateTime.Now;
            IndexLoader.MergeOrUpload(providerIndexList, apiKey, serviceName, "providers", 500);
            Console.WriteLine($"Index uploaded.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
        }
Esempio n. 6
0
        public static void Upload(string apiKey, string serviceName)
        {
            DateTime startDateTime = DateTime.Now;

            //Get all the provider data.
            List <KyruusDataStructure> providers = ProviderDa.GetAllFromDownload();

            Console.WriteLine($"{providers.Count} providers fetched.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Get the complete list of names.
            startDateTime = DateTime.Now;
            List <string> firstAndLastNames = new List <string>();

            foreach (KyruusDataStructure p in providers)
            {
                if (p.name == null)
                {
                    continue;
                }
                string firstName        = string.Empty;
                string lastName         = string.Empty;
                string firstAndLastName = string.Empty;
                //p.name.full_name does not have the p.preferred_name in the first name part.
                if (string.IsNullOrWhiteSpace(p.preferred_name))
                {
                    firstName = p.name.first_name;
                }
                else
                {
                    firstName = p.preferred_name;
                }
                if (string.IsNullOrWhiteSpace(firstName))
                {
                    firstName = string.Empty;
                }
                lastName = p.name.last_name;
                if (string.IsNullOrWhiteSpace(lastName))
                {
                    lastName = string.Empty;
                }
                if (firstName.Length == 0)
                {
                    firstAndLastName = lastName;
                }
                else
                {
                    firstAndLastName = firstName + " " + lastName;
                }
                firstAndLastNames.Add(firstAndLastName);
            }
            Console.WriteLine($"{firstAndLastNames.Count} names.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //De-dupe the names.
            startDateTime     = DateTime.Now;
            firstAndLastNames = firstAndLastNames
                                .Where(p => string.IsNullOrWhiteSpace(p) == false)
                                .Distinct()
                                .ToList();
            //Now assign the ID
            NameIndexDataStructure[] names = new NameIndexDataStructure[firstAndLastNames.Count];
            for (int n = 0; n < firstAndLastNames.Count; n++)
            {
                names[n] = new NameIndexDataStructure {
                    firstAndLastName = firstAndLastNames[n], id = (n + 1).ToString()
                };
            }
            Console.WriteLine($"{names.Length} names to be uploaded.  Response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Drop and recreate the Azure Index.
            startDateTime = DateTime.Now;
            RecreateIndex();
            Console.WriteLine($"Recreate index response time {(DateTime.Now - startDateTime).TotalMilliseconds}");

            //Populate index.
            startDateTime = DateTime.Now;
            IndexLoader.MergeOrUpload(names.ToList(), apiKey, serviceName, "names");
            Console.WriteLine($"Index upload response time {(DateTime.Now - startDateTime).TotalMilliseconds}");
        }