public void ReindexMinimal()
{
var toIndex = ElasticsearchConfiguration.NewUniqueIndexName();
var observable = this.Client.Reindex<object>(r => r
.FromIndex(ElasticsearchConfiguration.DefaultIndex)
.ToIndex(toIndex)
);
var observer = new ReindexObserver<object>(
onError: (e) => Assert.Fail(e.Message),
completed: () =>
{
var refresh = this.Client.Refresh(r=>r.Indices(toIndex, ElasticsearchConfiguration.DefaultIndex));
var originalIndexCount = this.Client.Count(c=>c
.Index(ElasticsearchConfiguration.DefaultIndex)
.Query(q => q.MatchAll())
);
var newIndexCount = this.Client.Count(c => c
.Index(toIndex)
.Query(q => q.MatchAll())
);
Assert.Greater(newIndexCount.Count, 0);
Assert.AreEqual(originalIndexCount.Count, newIndexCount.Count);
}
);
observable.Subscribe(observer);
}
public void ReturnsExpectedResponse()
{
var handles = new[]
{
new ManualResetEvent(false),
new ManualResetEvent(false)
};
var manyTypesObserver = new ReindexObserver <ILazyDocument>(
onError: (e) => { throw e; },
onCompleted: () => ReindexManyTypesCompleted(handles),
alter: (h, d, i) =>
{
//This would be a great place to alter the
// BulkIndexOperation i (use pipeline perhaps)
// Source Document d
// h is the whole hit including the documents metadata
//
// on a per document basis
i.RetriesOnConflict = 2;
}
);
this._reindexManyTypesResult.Subscribe(manyTypesObserver);
var singleTypeObserver = new ReindexObserver <Project>(
onError: (e) => { throw e; },
onCompleted: () => ReindexSingleTypeCompleted(handles)
);
WaitHandle.WaitAll(handles, TimeSpan.FromMinutes(3));
}
public void ReIndex()
{
const string newIndex = "cx_demo_claims_2";
var reindex =
_client.Reindex <ClaimModel>(Storage.ClaimIndex, newIndex, (inbox => inbox
.MatchAll()));
var observer = new ReindexObserver(onNext: response =>
{
Console.WriteLine($"Indexing documents");
}, onError: error =>
{
Console.WriteLine($"Document error occured: {error.Message}");
},
onCompleted: () =>
{
Console.WriteLine($"Document re-index");
});
reindex.Subscribe(observer);
_client.DeleteIndex(Indices.Index(new IndexName {
Name = Storage.ClaimIndex
}));
_client.Alias(x => x.Add(a => a.Alias(Storage.ClaimsAlias).Index(newIndex)));
}
public void ReindexMinimal()
{
var toIndex = ElasticsearchConfiguration.NewUniqueIndexName();
var observable = this.Client.Reindex <object>(r => r
.FromIndex(ElasticsearchConfiguration.DefaultIndex)
.ToIndex(toIndex)
);
var observer = new ReindexObserver <object>(
onError: (e) => Assert.Fail(e.Message),
completed: () =>
{
var refresh = this.Client.Refresh(r => r.Indices(toIndex, ElasticsearchConfiguration.DefaultIndex));
var originalIndexCount = this.Client.Count(c => c
.Index(ElasticsearchConfiguration.DefaultIndex)
.Query(q => q.MatchAll())
);
var newIndexCount = this.Client.Count(c => c
.Index(toIndex)
.Query(q => q.MatchAll())
);
Assert.Greater(newIndexCount.Count, 0);
Assert.AreEqual(originalIndexCount.Count, newIndexCount.Count);
}
);
observable.Subscribe(observer);
}
[I] public void ReturnsExpectedResponse()
{
var observableWait = new CountdownEvent(3);
Exception ex = null;
var manyTypesObserver = new ReindexObserver(
onError: (e) => { ex = e; observableWait.Signal(); throw e; },
onCompleted: () => ReindexManyTypesCompleted(observableWait)
);
this._reindexManyTypesResult.Subscribe(manyTypesObserver);
var singleTypeObserver = new ReindexObserver(
onError: (e) => { ex = e; observableWait.Signal(); throw e; },
onCompleted: () => ReindexSingleTypeCompleted(observableWait)
);
this._reindexSingleTypeResult.Subscribe(singleTypeObserver);
var projectionObserver = new ReindexObserver(
onError: (e) => { ex = e; observableWait.Signal(); throw e; },
onCompleted: () => ProjectionCompleted(observableWait)
);
this._reindexProjectionResult.Subscribe(projectionObserver);
observableWait.Wait(TimeSpan.FromMinutes(3));
if (ex != null)
{
throw ex;
}
}
public void ReindexSingleType(Func <CountdownEvent> getCountDown, Action <Exception> signal)
{
var singleTypeObserver = new ReindexObserver(
onError: signal,
onCompleted: () => ReindexSingleTypeCompleted(getCountDown())
);
_reindexSingleTypeResult.Subscribe(singleTypeObserver);
}
public void ReindexProjection(Func <CountdownEvent> getCountDown, Action <Exception> signal)
{
var projectionObserver = new ReindexObserver(
onError: signal,
onCompleted: () => ProjectionCompleted(getCountDown())
);
_reindexProjectionResult.Subscribe(projectionObserver);
}
public void ReindexMany(Func <CountdownEvent> getCountDown, Action <Exception> signal)
{
var manyTypesObserver = new ReindexObserver(
onError: signal,
onCompleted: () => ReindexManyTypesCompleted(getCountDown())
);
_reindexManyTypesResult.Subscribe(manyTypesObserver);
}
public void ReindexMany(Func <CountdownEvent> getCountDown, Action <Exception> signal)
{
var manyTypesObserver = new ReindexObserver(
onError: signal,
onCompleted: () => ReindexManyTypesCompleted(getCountDown())
);
this._reindexManyTypesResult.Subscribe(manyTypesObserver);
this._reindexManyTypesResult.Wait(TimeSpan.FromMinutes(3), r => { });
}
public void Reindex()
{
var toIndex = ElasticsearchConfiguration.NewUniqueIndexName();
var observable = this.Client.Reindex <object>(r => r
.FromIndex(ElasticsearchConfiguration.DefaultIndex)
.ToIndex(toIndex)
.Query(q => q.MatchAll())
.Scroll("10s")
.CreateIndex(c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
)
);
var observer = new ReindexObserver <object>(
onNext: (r) =>
{
var scrollResults = r.SearchResponse;
var bulkResults = r.BulkResponse;
Assert.NotNull(scrollResults);
Assert.NotNull(bulkResults);
Assert.True(scrollResults.IsValid);
Assert.True(bulkResults.IsValid);
},
onError: (e) => Assert.Fail(e.Message),
completed: () =>
{
var refresh = this.Client.Refresh(r => r.Indices(toIndex, ElasticsearchConfiguration.DefaultIndex));
var originalIndexCount = this.Client.Count(c => c
.Index(ElasticsearchConfiguration.DefaultIndex)
.Query(q => q.MatchAll())
);
var newIndexCount = this.Client.Count(i => i.Index(toIndex).Query(q => q.MatchAll()));
Assert.Greater(newIndexCount.Count, 0);
Assert.AreEqual(originalIndexCount.Count, newIndexCount.Count);
}
);
observable.Subscribe(observer);
}
public void Reindex()
{
var toIndex = ElasticsearchConfiguration.NewUniqueIndexName();
var observable = this.Client.Reindex<object>(r => r
.FromIndex(ElasticsearchConfiguration.DefaultIndex)
.ToIndex(toIndex)
.Query(q=>q.MatchAll())
.Scroll("10s")
.CreateIndex(c=>c
.NumberOfReplicas(0)
.NumberOfShards(1)
)
);
var observer = new ReindexObserver<object>(
onNext: (r) =>
{
var scrollResults = r.SearchResponse;
var bulkResults = r.BulkResponse;
Assert.NotNull(scrollResults);
Assert.NotNull(bulkResults);
Assert.True(scrollResults.IsValid);
Assert.True(bulkResults.IsValid);
},
onError: (e) => Assert.Fail(e.Message),
completed: () =>
{
var refresh = this.Client.Refresh(r=>r.Indices(toIndex, ElasticsearchConfiguration.DefaultIndex));
var originalIndexCount = this.Client.Count(c=>c
.Index(ElasticsearchConfiguration.DefaultIndex)
.Query(q=>q.MatchAll())
);
var newIndexCount = this.Client.Count(i=>i.Index(toIndex).Query(q=>q.MatchAll()));
Assert.Greater(newIndexCount.Count, 0);
Assert.AreEqual(originalIndexCount.Count, newIndexCount.Count);
}
);
observable.Subscribe(observer);
}
/// <summary>
/// Maintenance call, used in order to update the current index into a new version
/// </summary>
/// <param name="removeOldIndex">If true older index get deleted</param>
/// <param name="errorLogger">Action used by the user to parse error messages during reindexing</param>
public virtual void ReIndex <T>(bool removeOldIndex = true, Action <string> errorLogger = null) where T : EsDocument
{
bool hasErrors = false;
var esClient = Manager.EsClient;
var currentIndexname = esClient.GetIndexByAlias(IndexAlias);
//var currentIndexname = index.Indices.Keys.FirstOrDefault();
if (currentIndexname == DocumentIndex)
{
throw new Exception("Trying to reindex same Index");
}
//Create new index mapping
createIndex <T>(currentIndexname);
//start reindexing
var reindex = esClient.Reindex <T>(currentIndexname, DocumentIndex, q => q.MatchAll());
var o = new ReindexObserver(onError: e =>
{
hasErrors = true;
if (errorLogger != null)
{
var errorMsg = string.Format("Failed to reindex:\r\n{0}\r\nDetails:\r\n{1}\r\n{2}\r\n{3}", e.Data, e.Message, e.InnerException, e.StackTrace);
errorLogger(errorMsg);
}
;
});
reindex.Subscribe(o);
//Change allias pointing
esClient.Indices.PutAlias(DocumentIndex, IndexAlias);
//esClient.Indices.PutAlias(DocumentIndex, IndexAlias);
//Alias(a => a
//.Add(aa => aa.Alias(IndexAlias).Index(DocumentIndex))
//.Remove(aa => aa.Alias(IndexAlias).Index(currentIndexname.Name)));
// Add completion flag
if (!hasErrors)
{// Delete if successfull old index
if (removeOldIndex)
{
esClient.Indices.Delete(new DeleteIndexRequest(currentIndexname));
}
}
}
[I] public void ReturnsExpectedResponse()
{
var handle = new ManualResetEvent(false);
var observer = new ReindexObserver <ILazyDocument>(
onError: (e) => { throw e; },
completed: () =>
{
var refresh = this._client.Refresh("dev-copy");
var originalIndexCount = this._client.Count <Developer>();
var newIndexCount = this._client.Count <Developer>(c => c.Index("dev-copy"));
originalIndexCount.Count.Should().BeGreaterThan(0).And.Be(newIndexCount.Count);
handle.Set();
}
);
this._reindexResult.Subscribe(observer);
handle.WaitOne(TimeSpan.FromMinutes(3));
//await this._reindexResult;
}
/**
* `Reindex` has an overload that accepts a function for how source documents should be mapped to destination documents. In addition,
* further control over reindexing can be achieved by using an observer to subscribe to the reindexing process to take some action on
* each successful bulk response, when an error occurs, and when the process has finished. The following example demonstrates these
* features.
*
* IMPORTANT: An observer should not throw exceptions from its interface implementations, such
* as `OnNext` and `OnError`. Any exceptions thrown should be expected to go unhandled. In light of this, any exception
* that occurs during the reindex process should be captured and thrown outside of the observer, as demonstrated in the
* example below. Take a look at the
* https://docs.microsoft.com/en-us/dotnet/standard/events/observer-design-pattern-best-practices#handling-exceptions[Observer Design Pattern best practices]
* on handling exceptions.
*/
public void ReindexWithObserver()
{
var reindexObservable = client.Reindex <Person, Person>(
person => person, // <1> a function to define how source documents are mapped to destination documents
r => r
.ScrollAll("5s", Environment.ProcessorCount, s => s
.Search(ss => ss
.Index("source_index")
)
)
.BulkAll(b => b
.Index("destination_index")
)
);
var waitHandle = new ManualResetEvent(false);
ExceptionDispatchInfo exceptionDispatchInfo = null;
var observer = new ReindexObserver(
onNext: response =>
{
// do something e.g. write number of pages to console
},
onError: exception =>
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(exception);
waitHandle.Set();
},
onCompleted: () => waitHandle.Set());
reindexObservable.Subscribe(observer); // <2> Subscribe to the observable, which will initiate the reindex process
waitHandle.WaitOne(); // <3> Block the current thread until a signal is received
exceptionDispatchInfo?.Throw(); // <4> If an exception was captured during the reindex process, throw it
}
[I] public void ReturnsExpectedResponse()
{
var handles = new[]
{
new ManualResetEvent(false),
new ManualResetEvent(false)
};
var manyTypesObserver = new ReindexObserver <ILazyDocument>(
onError: (e) => { throw e; },
completed: () =>
{
var refresh = this._client.Refresh(NewManyTypesIndexName);
var originalIndexCount = this._client.Count <CommitActivity>(c => c.Index(IndexName));
var newIndexCount = this._client.Count <CommitActivity>(c => c.Index(NewManyTypesIndexName));
originalIndexCount.Count.Should().BeGreaterThan(0).And.Be(newIndexCount.Count);
var scroll = "20s";
var searchResult = this._client.Search <CommitActivity>(s => s
.Index(NewManyTypesIndexName)
.From(0)
.Size(100)
.Query(q => q.MatchAll())
.SearchType(SearchType.Scan)
.Scroll(scroll)
);
do
{
var result = searchResult;
searchResult = this._client.Scroll <CommitActivity>(scroll, result.ScrollId);
foreach (var hit in searchResult.Hits)
{
hit.Timestamp.Should().HaveValue();
hit.Parent.Should().NotBeNullOrEmpty();
hit.Routing.Should().NotBeNullOrEmpty();
}
} while (searchResult.IsValid && searchResult.Documents.Any());
handles[0].Set();
}
);
this._reindexManyTypesResult.Subscribe(manyTypesObserver);
var singleTypeObserver = new ReindexObserver <Project>(
onError: (e) => { throw e; },
completed: () =>
{
var refresh = this._client.Refresh(NewSingleTypeIndexName);
var originalIndexCount = this._client.Count <Project>(c => c.Index(IndexName));
// new index should only contain project document types
var newIndexCount = this._client.Count <Project>(c => c.Index(NewSingleTypeIndexName).AllTypes());
originalIndexCount.Count.Should().BeGreaterThan(0).And.Be(newIndexCount.Count);
handles[1].Set();
}
);
this._reindexSingleTypeResult.Subscribe(singleTypeObserver);
WaitHandle.WaitAll(handles, TimeSpan.FromMinutes(3));
}
