public ReindexResponse Post(IRestClient client, ReindexRequest reindexRequest, out string resourceCall) { var resource = "_reindex"; resourceCall = $"{client.BaseUrl}{resource}"; var request = new RestRequest(resource, Method.POST) { RequestFormat = DataFormat.Json }; request.AddHeader("Accept", "application/json"); request.AddJsonBody(reindexRequest); var response = client.Execute <ReindexResponse>(request); if (response.IsSuccessful) { return(response.Data); } throw new RestCallException(response.StatusCode, response.StatusDescription, response.ErrorMessage, response.ErrorException); }
private static void ProcessIndexes() { var client = RestClientUtility.Instance.NewClient(_commandLineModel.Url, _IAuth); var catApi = new CatAPI(); var reindexApi = new ReindexAPI(); var indexApi = new IndexAPI(); var searchAndScrollApi = new SearchAndScrollAPI(); var bulkApi = new BulkAPI(); var indexes = catApi.Get(client, _commandLineModel.SourceIndexPattern, out _currentResourceCall); indexes = indexes.OrderBy(m => m.index).ToList(); if (_commandLineModel.Verbose) { VerboseLogLine($"Call to '{_currentResourceCall} returned {indexes.Count} indexes."); } var sw = new Stopwatch(); foreach (var index in indexes) { string verb; if (_commandLineModel.NoDestinationJustDelete) { verb = _commandLineModel.DryRun ? "Deleting DryRun" : "Deleting"; } else { verb = _commandLineModel.DryRun ? "Reindexing DryRun" : "Reindexing"; } var rir = new ReindexRequest { conflicts = "proceed", source = new ReindexSource() { index = index.index }, dest = new ReindexDestination() { index = BuildDestinationIndex(index.index), op_type = "create" } }; var reindexResultModel = new ReindexResultModel() { IndexSource = rir.source.index, IndexDestination = rir.dest.index }; try { var skipIndex = false; if (!index.health.CompareNoCase("green") && !index.health.CompareNoCase("yellow")) { reindexResultModel.ReindexSkipped = true; reindexResultModel.SkippedReason = $"Index health '{index.health}'."; skipIndex = true; } else if (!index.status.CompareNoCase("open")) { reindexResultModel.ReindexSkipped = true; reindexResultModel.SkippedReason = $"Index status '{index.health}'."; skipIndex = true; } else if (index.docscount == 0) { reindexResultModel.ReindexSkipped = true; reindexResultModel.SkippedReason = "No documents in index."; skipIndex = true; } if (skipIndex) { if (_commandLineModel.DryRun || _commandLineModel.Verbose) { VerboseLogLine($"{verb} ({index.docscount} Docs):"); VerboseLogLine($" {index.index} ! SKIPPED ({reindexResultModel.SkippedReason})"); } } else { if (_commandLineModel.DryRun || _commandLineModel.Verbose) { VerboseLogLine($"{verb} ({index.docscount} Docs):"); VerboseLogLine($" {index.index} ==> {rir.dest.index}"); } if (!_commandLineModel.DryRun) { // Can we use reindex API or do we need to scroll? if (_commandLineModel.ScrollCount == 0 || _commandLineModel.ScrollCount >= index.docscount | _commandLineModel.NoDestinationJustDelete) { reindexResultModel.DocsInOldIndex = index.docscount; sw.Restart(); if (_commandLineModel.NoDestinationJustDelete) { reindexResultModel.DocsDeleted = index.docscount; var dir = indexApi.Delete(client, rir.source.index, out _currentResourceCall); reindexResultModel.DeleteAcknowledged = dir.acknowledged; } else { var reindexResponse = reindexApi.Post(client, rir, out _currentResourceCall); reindexResultModel.DocsCreated = reindexResponse.created; reindexResultModel.DocsUpdated = reindexResponse.updated; reindexResultModel.DocsVersionConflicts = reindexResponse.version_conflicts; if (reindexResponse.created + reindexResponse.updated + reindexResponse.version_conflicts != index.docscount) { reindexResultModel.DeleteIndexSkipped = true; VerboseLogLine($"WARNING: Skipped call delete index '{index.index}' as reindex document count does not match source count (DestCreated {reindexResponse.created} + DestUpdated {reindexResponse.updated} + DestVerConflict {reindexResponse.version_conflicts} != Src {index.docscount})"); } else if (!_commandLineModel.CopyOnly) { var dir = indexApi.Delete(client, rir.source.index, out _currentResourceCall); reindexResultModel.DeleteAcknowledged = dir.acknowledged; } } sw.Stop(); reindexResultModel.ActionTime = sw.ElapsedMilliseconds; } // We need to scroll rather than reindex. else { reindexResultModel.DocsInOldIndex = index.docscount; sw.Restart(); var sasRequest = new SearchAndScrollRequest() { size = _commandLineModel.ScrollCount, query = new MatchAllQuery() }; var searchAndScrollResponse = searchAndScrollApi.PostSearch(client, sasRequest, rir.source.index, "5m", out _currentResourceCall); while (searchAndScrollResponse.hits.hits.Count > 0) { var bulkRequestBuilder = new StringBuilder(); foreach (var hit in searchAndScrollResponse.hits.hits) { bulkRequestBuilder.AddHitToBulkOperation(IndexDocumentActionType.Create, hit._source, rir.dest.index, hit._type, hit._id); if (!_commandLineModel.CopyOnly) { bulkRequestBuilder.AddHitToBulkOperation(IndexDocumentActionType.Delete, hit._source, hit._index, hit._type, hit._id); } } var bulkRequestBody = bulkRequestBuilder.ToString(); var bulkResponse = bulkApi.Post(client, bulkRequestBody, out _currentResourceCall); if (bulkResponse.errors == true) { reindexResultModel.ExceptionEncountered = true; } reindexResultModel.DocsCreated += bulkResponse.items.Where(m => m.create != null && (m.create.status == 201)).ToList().Count; searchAndScrollResponse = searchAndScrollApi.PostScroll(client, "5m", searchAndScrollResponse.scroll_id, out _currentResourceCall); } if (reindexResultModel.DocsCreated != reindexResultModel.DocsInOldIndex) { reindexResultModel.DeleteIndexSkipped = true; VerboseLogLine($"WARNING: Skipped call delete index '{index.index}' as document copy count does not match source count (DestCreated {reindexResultModel.DocsCreated} != Src {index.docscount})"); } else if (!_commandLineModel.CopyOnly) { var dir = indexApi.Delete(client, rir.source.index, out _currentResourceCall); reindexResultModel.DeleteAcknowledged = dir.acknowledged; } sw.Stop(); reindexResultModel.ActionTime = sw.ElapsedMilliseconds; } } else { reindexResultModel.ReindexSkipped = true; reindexResultModel.SkippedReason = "DryRun is true"; } } } catch (RestCallException rex) { VerboseLogLine($"ERROR: RESTFul Call Failed calling '{_currentResourceCall}'"); VerboseLogLine($"Http Status Code: {rex.StatusCode}"); VerboseLogLine($"Http Status Description: {rex.StatusDescription}"); if (rex.InnerException != null) { VerboseLogLine($"Exception: {rex.InnerException.Message}"); } VerboseLogLine(""); reindexResultModel.ExceptionEncountered = true; reindexResultModel.ExceptionInfo = $"'{_currentResourceCall}' Call Failed, Status Code: {rex.StatusCode}, Status Description: {rex.StatusDescription}"; } _results.Add(reindexResultModel); } }