public void Init()
{
if (StartId == null)
{
StartId = long.MaxValue;
}
isFirstLoad = StartId == long.MaxValue;
pages = new List <FTServer.Page>();
begin = DateTime.Now;
StartId = IndexAPI.search(pages, Query, StartId.Value, pageCount);
if (StartId == long.MaxValue)
{
Page p = new Page();
p.title = "NotFound";
p.description = "";
p.content = "input URL(http or https) to index";
p.url = "./";
pages.Add(p);
}
}
private static void ProcessIndexes()
{
var client = RestClientUtility.Instance.NewClient(_commandLineModel.Url, _IAuth);
var catApi = new CatAPI();
var reindexApi = new ReindexAPI();
var indexApi = new IndexAPI();
var searchAndScrollApi = new SearchAndScrollAPI();
var bulkApi = new BulkAPI();
var indexes = catApi.Get(client, _commandLineModel.SourceIndexPattern, out _currentResourceCall);
indexes = indexes.OrderBy(m => m.index).ToList();
if (_commandLineModel.Verbose)
{
VerboseLogLine($"Call to '{_currentResourceCall} returned {indexes.Count} indexes.");
}
var sw = new Stopwatch();
foreach (var index in indexes)
{
string verb;
if (_commandLineModel.NoDestinationJustDelete)
{
verb = _commandLineModel.DryRun ? "Deleting DryRun" : "Deleting";
}
else
{
verb = _commandLineModel.DryRun ? "Reindexing DryRun" : "Reindexing";
}
var rir = new ReindexRequest
{
conflicts = "proceed",
source = new ReindexSource()
{
index = index.index
},
dest = new ReindexDestination()
{
index = BuildDestinationIndex(index.index),
op_type = "create"
}
};
var reindexResultModel = new ReindexResultModel()
{
IndexSource = rir.source.index,
IndexDestination = rir.dest.index
};
try
{
var skipIndex = false;
if (!index.health.CompareNoCase("green") && !index.health.CompareNoCase("yellow"))
{
reindexResultModel.ReindexSkipped = true;
reindexResultModel.SkippedReason = $"Index health '{index.health}'.";
skipIndex = true;
}
else if (!index.status.CompareNoCase("open"))
{
reindexResultModel.ReindexSkipped = true;
reindexResultModel.SkippedReason = $"Index status '{index.health}'.";
skipIndex = true;
}
else if (index.docscount == 0)
{
reindexResultModel.ReindexSkipped = true;
reindexResultModel.SkippedReason = "No documents in index.";
skipIndex = true;
}
if (skipIndex)
{
if (_commandLineModel.DryRun || _commandLineModel.Verbose)
{
VerboseLogLine($"{verb} ({index.docscount} Docs):");
VerboseLogLine($" {index.index} ! SKIPPED ({reindexResultModel.SkippedReason})");
}
}
else
{
if (_commandLineModel.DryRun || _commandLineModel.Verbose)
{
VerboseLogLine($"{verb} ({index.docscount} Docs):");
VerboseLogLine($" {index.index} ==> {rir.dest.index}");
}
if (!_commandLineModel.DryRun)
{
// Can we use reindex API or do we need to scroll?
if (_commandLineModel.ScrollCount == 0 || _commandLineModel.ScrollCount >= index.docscount | _commandLineModel.NoDestinationJustDelete)
{
reindexResultModel.DocsInOldIndex = index.docscount;
sw.Restart();
if (_commandLineModel.NoDestinationJustDelete)
{
reindexResultModel.DocsDeleted = index.docscount;
var dir = indexApi.Delete(client, rir.source.index, out _currentResourceCall);
reindexResultModel.DeleteAcknowledged = dir.acknowledged;
}
else
{
var reindexResponse = reindexApi.Post(client, rir, out _currentResourceCall);
reindexResultModel.DocsCreated = reindexResponse.created;
reindexResultModel.DocsUpdated = reindexResponse.updated;
reindexResultModel.DocsVersionConflicts = reindexResponse.version_conflicts;
if (reindexResponse.created + reindexResponse.updated +
reindexResponse.version_conflicts != index.docscount)
{
reindexResultModel.DeleteIndexSkipped = true;
VerboseLogLine($"WARNING: Skipped call delete index '{index.index}' as reindex document count does not match source count (DestCreated {reindexResponse.created} + DestUpdated {reindexResponse.updated} + DestVerConflict {reindexResponse.version_conflicts} != Src {index.docscount})");
}
else if (!_commandLineModel.CopyOnly)
{
var dir = indexApi.Delete(client, rir.source.index, out _currentResourceCall);
reindexResultModel.DeleteAcknowledged = dir.acknowledged;
}
}
sw.Stop();
reindexResultModel.ActionTime = sw.ElapsedMilliseconds;
}
// We need to scroll rather than reindex.
else
{
reindexResultModel.DocsInOldIndex = index.docscount;
sw.Restart();
var sasRequest = new SearchAndScrollRequest()
{
size = _commandLineModel.ScrollCount,
query = new MatchAllQuery()
};
var searchAndScrollResponse = searchAndScrollApi.PostSearch(client, sasRequest, rir.source.index, "5m", out _currentResourceCall);
while (searchAndScrollResponse.hits.hits.Count > 0)
{
var bulkRequestBuilder = new StringBuilder();
foreach (var hit in searchAndScrollResponse.hits.hits)
{
bulkRequestBuilder.AddHitToBulkOperation(IndexDocumentActionType.Create, hit._source, rir.dest.index, hit._type, hit._id);
if (!_commandLineModel.CopyOnly)
{
bulkRequestBuilder.AddHitToBulkOperation(IndexDocumentActionType.Delete, hit._source, hit._index, hit._type, hit._id);
}
}
var bulkRequestBody = bulkRequestBuilder.ToString();
var bulkResponse = bulkApi.Post(client, bulkRequestBody, out _currentResourceCall);
if (bulkResponse.errors == true)
{
reindexResultModel.ExceptionEncountered = true;
}
reindexResultModel.DocsCreated += bulkResponse.items.Where(m => m.create != null && (m.create.status == 201)).ToList().Count;
searchAndScrollResponse = searchAndScrollApi.PostScroll(client, "5m", searchAndScrollResponse.scroll_id, out _currentResourceCall);
}
if (reindexResultModel.DocsCreated != reindexResultModel.DocsInOldIndex)
{
reindexResultModel.DeleteIndexSkipped = true;
VerboseLogLine($"WARNING: Skipped call delete index '{index.index}' as document copy count does not match source count (DestCreated {reindexResultModel.DocsCreated} != Src {index.docscount})");
}
else if (!_commandLineModel.CopyOnly)
{
var dir = indexApi.Delete(client, rir.source.index, out _currentResourceCall);
reindexResultModel.DeleteAcknowledged = dir.acknowledged;
}
sw.Stop();
reindexResultModel.ActionTime = sw.ElapsedMilliseconds;
}
}
else
{
reindexResultModel.ReindexSkipped = true;
reindexResultModel.SkippedReason = "DryRun is true";
}
}
}
catch (RestCallException rex)
{
VerboseLogLine($"ERROR: RESTFul Call Failed calling '{_currentResourceCall}'");
VerboseLogLine($"Http Status Code: {rex.StatusCode}");
VerboseLogLine($"Http Status Description: {rex.StatusDescription}");
if (rex.InnerException != null)
{
VerboseLogLine($"Exception: {rex.InnerException.Message}");
}
VerboseLogLine("");
reindexResultModel.ExceptionEncountered = true;
reindexResultModel.ExceptionInfo = $"'{_currentResourceCall}' Call Failed, Status Code: {rex.StatusCode}, Status Description: {rex.StatusDescription}";
}
_results.Add(reindexResultModel);
}
}
public static async Task <String> IndexTextAsync(String name, bool onlyDelete)
{
return(await IndexAPI.indexTextAsync(name, onlyDelete));
}
