public void Index(TSourceSummary vacancySummaryToIndex) { _logger.Debug("Indexing vacancy item : {0} ({1})", vacancySummaryToIndex.Title, vacancySummaryToIndex.Id); try { var indexAlias = GetIndexAlias(); var newIndexName = GetIndexNameAndDateExtension(indexAlias, vacancySummaryToIndex.ScheduledRefreshDateTime, vacancySummaryToIndex.UseAlias); var vacancySummaryElastic = _mapper.Map <TSourceSummary, TDestinationSummary>(vacancySummaryToIndex); var client = _elasticsearchClientFactory.GetElasticClient(); var indexResponse = client.Index(vacancySummaryElastic, f => f.Index(newIndexName)); if (!indexResponse.ConnectionStatus.Success) { _logger.Info("Failed to index vacancy item : {0} ({1}). Response {2} {3}", vacancySummaryToIndex.Title, vacancySummaryToIndex.Id, indexResponse.ConnectionStatus.HttpStatusCode, indexResponse.ConnectionStatus); } else { _logger.Debug("Indexed vacancy item : {0} ({1})", vacancySummaryToIndex.Title, vacancySummaryToIndex.Id); } } catch (Exception ex) { _logger.Warn("Failed indexing traineeship vacancy summary {0}", ex, vacancySummaryToIndex.Id); } }
public SearchResults <TraineeshipSearchResponse, TraineeshipSearchParameters> FindVacancies(TraineeshipSearchParameters parameters) { var client = _elasticsearchClientFactory.GetElasticClient(); var indexName = _elasticsearchClientFactory.GetIndexNameForType(typeof(TraineeshipSummary)); var documentTypeName = _elasticsearchClientFactory.GetDocumentNameForType(typeof(TraineeshipSummary)); _logger.Debug("Calling legacy vacancy search for DocumentNameForType={0} on IndexName={1}", documentTypeName, indexName); var search = PerformSearch(parameters, client, indexName, documentTypeName); var responses = _vacancySearchMapper.Map <IEnumerable <TraineeshipSummary>, IEnumerable <TraineeshipSearchResponse> >(search.Documents).ToList(); responses.ForEach(r => { var hitMd = search.HitsMetaData.Hits.First(h => h.Id == r.Id.ToString(CultureInfo.InvariantCulture)); if (parameters.Location != null) { if (parameters.SortType == VacancySearchSortType.ClosingDate || parameters.SortType == VacancySearchSortType.Distance || parameters.SortType == VacancySearchSortType.RecentlyAdded) { r.Distance = double.Parse(hitMd.Sorts.Skip(hitMd.Sorts.Count() - 1).First().ToString()); } } r.Score = hitMd.Score; }); _logger.Debug("{0} search results returned", search.Total); var results = new SearchResults <TraineeshipSearchResponse, TraineeshipSearchParameters>(search.Total, responses, null, parameters); return(results); }
public SearchResults <ApprenticeshipSearchResponse, ApprenticeshipSearchParameters> FindVacancies(ApprenticeshipSearchParameters parameters, string indexName) { var client = _elasticsearchClientFactory.GetElasticClient(); var documentTypeName = _elasticsearchClientFactory.GetDocumentNameForType(typeof(ApprenticeshipSummary)); _logger.Debug("Calling legacy vacancy search for DocumentNameForType={0} on IndexName={1}", documentTypeName, indexName); ProcessSearchParameters(parameters); var search = PerformSearch(parameters, client, indexName, documentTypeName); var responses = _vacancySearchMapper.Map <IEnumerable <ApprenticeshipSummary>, IEnumerable <ApprenticeshipSearchResponse> > (search.Documents).ToList(); responses.ForEach(r => { var hitMd = search.HitsMetaData.Hits.First(h => h.Id == r.Id.ToString(CultureInfo.InvariantCulture)); if (parameters.Location != null) { if (parameters.SortType == VacancySearchSortType.ClosingDate || parameters.SortType == VacancySearchSortType.Distance || parameters.SortType == VacancySearchSortType.RecentlyAdded) { r.Distance = double.Parse(hitMd.Sorts.Skip(hitMd.Sorts.Count() - 1).First().ToString()); } else { //if anyone can find a better way to get this value out, feel free! var array = hitMd.Fields.FieldValues <JArray>("distance"); var value = array[0]; r.Distance = double.Parse(value.ToString()); } } r.Score = hitMd.Score; }); _logger.Debug("{0} search results returned", search.Total); var aggregationResults = GetAggregationResultsFrom(search.Aggs); var results = new SearchResults <ApprenticeshipSearchResponse, ApprenticeshipSearchParameters>(search.Total, responses, aggregationResults, parameters); return(results); }
public IEnumerable <Address> FindAddress(string postcode) { Condition.Requires(postcode, "postcode").IsNotNullOrWhiteSpace(); _logger.Debug("FindAddress for postcode {0}", postcode); var client = _elasticsearchClientFactory.GetElasticClient(); var indexName = _elasticsearchClientFactory.GetIndexNameForType(typeof(Elastic.Common.Entities.Address)); var documentTypeName = _elasticsearchClientFactory.GetDocumentNameForType(typeof(Elastic.Common.Entities.Address)); var postcodeSearch = postcode.Replace(" ", ""); var search = client.Search <Elastic.Common.Entities.Address>(s => { s.Index(indexName); s.Type(documentTypeName); s.Size(100); s.Query(q => q.MatchPhrase(mpqd => mpqd.OnField(a => a.PostcodeSearch).Query(postcodeSearch))); return(s); }); //TODO: vga: refactor in a new ElasticClient class if (ThereWasAnErrorWhileSearching(search)) { throw search.ConnectionStatus.OriginalException; } var addresses = _mapper .Map <IEnumerable <Elastic.Common.Entities.Address>, IEnumerable <Address> >(search.Documents) .ToList(); SanitiseAddresses(addresses); return(addresses); }
public void Run() { var client = _elasticsearchClientFactory.GetElasticClient(); var expectedAliasNames = GetExpectedAliasNames(); EnsureExpectedAliasesExist(client, expectedAliasNames); }
public void Run() { _logger.Info("Getting draft vacancy ids"); var draftVacancyIds = _applicationDiagnosticsRepository.GetDraftApplicationVacancyIds().ToList(); var draftVacancyIdsString = string.Join(", ", draftVacancyIds); _logger.Info("Draft vacancy ids {0}: {1}", draftVacancyIds.Count, draftVacancyIdsString); var client = _elasticsearchClientFactory.GetElasticClient(); var indexName = _elasticsearchClientFactory.GetIndexNameForType(typeof(Elastic.Common.Entities.ApprenticeshipSummary)); _logger.Info("Getting vacancy ids in index"); var searchResponse = client.Search <Elastic.Common.Entities.ApprenticeshipSummary>(s => s .Index(indexName) .Fields("id") .Filter(fc => fc.Ids(draftVacancyIds)) .Size(draftVacancyIds.Count)); var vacancyIdsInIndex = searchResponse.Hits.Select(v => v.Id).ToList(); var vacancyIdsInIndexString = string.Join(", ", vacancyIdsInIndex); _logger.Info("Vacancy ids in index {0}: {1}", vacancyIdsInIndex.Count, vacancyIdsInIndexString); var expiredVacancyIds = draftVacancyIds.Except(vacancyIdsInIndex).ToList(); var expiredVacancyIdsString = string.Join(", ", expiredVacancyIds); _logger.Info("Expired vacancy ids {0}: {1}", expiredVacancyIds.Count, expiredVacancyIdsString); /*var verifiedExpiredVacancyIds = expiredVacancyIds.ToList(); * * Logger.Info("Verifying expired vacancies"); * foreach (var expiredVacancyId in expiredVacancyIds) * { * int vacancyId; * if (int.TryParse(expiredVacancyId, out vacancyId)) * { * var vacancyDetails = _vacancySearchService.GetVacancyDetails(vacancyId); * if (vacancyDetails != null) * { * Logger.Info("Vacancy with id: {0} has not expired and will be removed", vacancyId); * verifiedExpiredVacancyIds.Remove(expiredVacancyId); * } * } * else * { * Logger.Info("Vacancy id: {0} could not be parsed so assume still valid", expiredVacancyId); * verifiedExpiredVacancyIds.Remove(expiredVacancyId); * } * } * * var verifiedExpiredVacancyIdsString = string.Join(", ", verifiedExpiredVacancyIds); * Logger.Info("Verified expired vacancy ids {0}: {1}", verifiedExpiredVacancyIds.Count, verifiedExpiredVacancyIdsString);*/ }
public long GetApprenticeshipsCount() { var client = _elasticsearchClientFactory.GetElasticClient(); var indexName = _elasticsearchClientFactory.GetIndexNameForType(typeof(ApprenticeshipSummary)); var documentTypeName = _elasticsearchClientFactory.GetDocumentNameForType(typeof(ApprenticeshipSummary)); var response = client.Count <ApprenticeshipSummary>(s => { s.Index(indexName); s.Type(documentTypeName); return(s); }); var count = response.Count; return(count); }
public void Index(TSourceSummary vacancySummaryToIndex) { _logger.Debug("Indexing vacancy item : {0} ({1})", vacancySummaryToIndex.Title, vacancySummaryToIndex.Id); try { var indexAlias = GetIndexAlias(); var newIndexName = GetIndexNameAndDateExtension(indexAlias, vacancySummaryToIndex.ScheduledRefreshDateTime); var vacancySummaryElastic = _mapper.Map <TSourceSummary, TDestinationSummary>(vacancySummaryToIndex); var client = _elasticsearchClientFactory.GetElasticClient(); client.Index(vacancySummaryElastic, f => f.Index(newIndexName)); _logger.Debug("Indexed vacancy item : {0} ({1})", vacancySummaryToIndex.Title, vacancySummaryToIndex.Id); } catch (Exception ex) { _logger.Warn("Failed indexing traineeship vacancy summary {0}", ex, vacancySummaryToIndex.Id); } }
private IHealthResponse GetClusterHealth() { var request = new ClusterHealthRequest { Level = Level.Cluster, //http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/cluster-health.html WaitForStatus = WaitForStatus.Yellow, Timeout = Timeout }; var client = _elasticsearchClientFactory.GetElasticClient(); return(client.ClusterHealth(request)); }
public IEnumerable <int> GetAllVacancyIds(string indexName) { _logger.Debug("Getting all vacancy id from index '{0}' with ScrollSize='{1}', ScrollIndexConsistencyTime='{2}', ScrollTimeout='{3}'", indexName, ScrollSize, ScrollIndexConsistencyTime, ScrollTimeout); var client = _elasticsearchClientFactory.GetElasticClient(); var documentTypeName = _elasticsearchClientFactory.GetDocumentNameForType(typeof(TVacancySummary)); // REFERENCE: http://nest.azurewebsites.net/nest/search/scroll.html. var scanResults = client.Search <TVacancySummary>(search => search .Index(indexName) .Type(documentTypeName) .From(0) .Size(ScrollSize) .MatchAll() .SearchType(SearchType.Scan) .Scroll(ScrollIndexConsistencyTime)); var vacancies = new List <int>(); var scrollRequest = new ScrollRequest(scanResults.ScrollId, ScrollTimeout); var scrollCount = 0; while (true) { var results = client.Scroll <TVacancySummary>(scrollRequest); scrollCount++; if (!results.Documents.Any()) { break; } vacancies.AddRange(results.Documents.Select(each => each.Id)); } _logger.Debug("Got {0} vacancy ids from index '{1}' in {2} 'scrolls'", vacancies.Count, indexName, scrollCount); return(vacancies); }
public ElasticsearchCustomClient(IElasticsearchClientFactory elasticsearchClientFactory, ILog logger) { _client = elasticsearchClientFactory.GetElasticClient(); _logger = logger; }