public async IAsyncEnumerable <IndexKey <TKeyType> > ReadIndexKeys <TDataType, TKeyType>(Expression <Func <TDataType, TKeyType> > indexProperty) where TDataType : DataEntity
{
var indexMember = ((MemberExpression)indexProperty.Body).Member;
using (Logger.BeginScope("{Operation} is {Action} {IndexName} for {DataType}",
nameof(DataManager), "getting keys from index", indexMember.Name, typeof(TDataType).Name))
{
if (indexMember.IsDefined(typeof(IndexAttribute), true))
{
Logger.LogTrace("Property is indexed");
var indexName = indexMember.Name;
if (await IndexWorker.IndexExist <TDataType>(indexName))
{
var keys = IndexWorker.GetKeysFromIndex <TDataType, TKeyType>(indexName);
Logger.LogInformation("Found keys in index", keys);
await foreach (var key in keys)
{
yield return(key);
}
}
yield break;
}
Logger.LogWarning("The property used is not marked as indexed");
}
}
public async IAsyncEnumerable <TDataType> ReadIndex <TDataType, TKeyType>(TKeyType key,
Expression <Func <TDataType, TKeyType> > indexProperty) where TDataType : DataEntity
{
var indexMember = ((MemberExpression)indexProperty.Body).Member;
using (Logger.BeginScope("{Operation} is {Action} {IndexName} for {DataType} with a key {key}",
nameof(DataManager), "read from index", indexMember.Name, typeof(TDataType).Name, key))
{
if (indexMember.IsDefined(typeof(IndexAttribute), true))
{
Logger.LogTrace("Property is indexed");
var indexName = indexMember.Name;
if (await IndexWorker.IndexExist <TDataType>(indexName))
{
Logger.LogTrace("Index exists");
var ids = IndexWorker.GetIdsFromIndex <TDataType, TKeyType>(indexName, key);
Logger.LogInformation("Found ids in index", ids, key);
await foreach (var id in ids)
{
yield return(await Read <TDataType>(id));
}
}
yield break;
}
Logger.LogWarning("The property used is not marked as indexed");
}
}
public IActionResult IndexAllShops(bool clean = true)
{
var context = new IndexAllShopsContext();
var worker = new IndexWorker(_settings, context, _works, _dbHelper);
return(_works.AddToQueue(worker.IndexAll, context, QueuePriority.Parallel, clean));
}
public IndexWorkerFixture()
{
_mockConfig = _mockRepo.Create <ITargetIndexingConfig>();
_mockEsClient = _mockRepo.Create <IEsIndexClient>();
_testWorker = new IndexWorker(_mockConfig.Object, _mockEsClient.Object);
}
protected override Action GetWorker(List <EsDocument> data)
{
return(new Action(() =>
{
using (var workerWrapper = _workerFactory.CreateReleasable((f) => f.Create()))
{
IndexWorker worker = workerWrapper;
worker.Index(data);
}
}));
}
public IActionResult IndexShop(
int id,
bool downloadFresh = false,
bool needLink = true,
bool needSoldOut = true,
bool clean = true)
{
var context = new IndexShopContext(id, downloadFresh, needLink, needSoldOut);
var worker = new IndexWorker(_settings, context, _works, _dbHelper);
return(_works.AddToQueue(worker.Index, context, QueuePriority.Parallel, clean));
}
public void BulkIndex_addsMultipleDoc_Successfully(
EsDocument doc1,
EsDocument doc2,
EsDocument doc3,
EsTestIndexClient testClient,
IndexWorker indexWorker)
{
indexWorker
.Index(new[] { doc1, doc2, doc3 })
.Should().Be(true);
using (testClient.ForTestAssertions())
{
testClient.GetAllDocs().Should().HaveCount(3);
}
}
public void BulkIndex_addsSingleDoc_Successfully(
EsDocument doc1,
EsTestIndexClient testClient,
IndexWorker indexWorker)
{
indexWorker
.Index(new List <EsDocument>()
{
doc1
})
.Should().Be(true);
using (testClient.ForTestAssertions())
{
testClient.GetAllDocs().Single().ToString().Should().Be(doc1.Data.ToString());
}
}
public void ScrollPage_MultipleDocsExistOnSingleScrollPage_ReturnsDocs(
EsDocument doc1,
EsDocument doc2,
EsDocument doc3,
IndexWorker indexWorker,
EsTestIndexClient testClient,
ScrollWorker scrollWorker)
{
indexWorker.Index(new [] { doc1, doc2, doc3 });
using (testClient.ForTestAssertions())
{
var actuals = scrollWorker.ScrollPage();
actuals.Should().NotBeNull();
actuals.Should().HaveCount(3);
}
}
public void ScrollPage_MultipleDocsExistOnMultipleScrollPages_ReturnsDocs(
IFixture fixture,
IndexWorker indexWorker,
EsTestIndexClient testClient,
ScrollWorker scrollWorker)
{
var testDocs = fixture.CreateMany <EsDocument>(scrollWorker.Client.Config.BatchSize * 2);
indexWorker.Index(testDocs);
using (testClient.ForTestAssertions())
{
var actuals = scrollWorker.ScrollPage(); // first page
actuals = actuals.Concat(scrollWorker.ScrollPage()); // second page
scrollWorker.ScrollPage().Should().BeEmpty(); // third page is empty
actuals.Should().NotBeNull();
actuals.Should().HaveSameCount(testDocs);
}
}
public async Task <bool> Restore <TDataType>(int id) where TDataType : DataEntity
{
using (Logger.BeginScope("{Operation} is {Action} {DataType} with Id ({Id})", nameof(DataManager),
"restoring", typeof(TDataType).Name, id))
{
var data = await Read <TDataType>(id);
if (data is null)
{
Logger.LogTrace("Data could be deleted");
if (await DataStreamer.Undelete <TDataType>(id))
{
Logger.LogTrace("Data restored to store");
data = await Read <TDataType>(id);
if (typeof(TDataType).IsDefined(typeof(AuditAttribute), true))
{
Logger.LogTrace("Data is audited");
if (await AuditWorker.UndeleteEvent(data))
{
if (await AuditWorker.CommitEvents(data))
{
Logger.LogTrace("Audit committed to store");
if (await IndexWorker.Index(data))
{
Logger.LogInformation("Restored data with audit");
return(true);
}
Logger.LogWarning("Failed to index data");
await AuditWorker.RollbackEvent(data);
}
else
{
Logger.LogWarning("Failed to commit audit for data to store");
await AuditWorker.DiscardEvents(data);
Logger.LogTrace("Discarded audit record");
}
}
else
{
Logger.LogWarning("Failed to create undelete audit record for data");
}
}
else
{
if (await IndexWorker.Index(data))
{
Logger.LogInformation("Restored data");
return(true);
}
Logger.LogWarning("Failed to index data");
}
await DataStreamer.Delete <TDataType>(id);
Logger.LogInformation("Rolled back restore");
}
Logger.LogWarning("Failed to restore data");
return(false);
}
Logger.LogWarning("Data didn't need restoring");
return(true);
}
}
public async Task <bool> Delete <TDataType>(int id) where TDataType : DataEntity
{
using (Logger.BeginScope("{Operation} is {Action} {DataType} with Id ({Id})", nameof(DataManager),
"deleting", typeof(TDataType).Name, id))
{
var data = await Read <TDataType>(id);
if (data is null)
{
Logger.LogWarning("Unable to find the data it may already be deleted");
return(false);
}
if (typeof(TDataType).IsDefined(typeof(AuditAttribute), true))
{
Logger.LogTrace("Data is audited");
if (await AuditWorker.DeleteEvent(data))
{
Logger.LogTrace("Delete audit record created");
if (await DataStreamer.Delete <TDataType>(id))
{
Logger.LogTrace("Data deleted from store");
if (await AuditWorker.CommitEvents(data))
{
Logger.LogTrace("Audit committed to store");
if (await IndexWorker.Unindex(data))
{
Logger.LogInformation("Data deleted from store");
return(true);
}
Logger.LogWarning("Failed to remove indexes for data");
await AuditWorker.RollbackEvent(data);
Logger.LogInformation("Added rollback audit record");
}
else
{
Logger.LogWarning("Failed to commit audit to store");
}
await DataStreamer.Rollback(id, data);
Logger.LogInformation("Rolled data back");
}
else
{
Logger.LogWarning("Failed to delete data");
await AuditWorker.DiscardEvents(data);
Logger.LogTrace("Discarded audit record");
}
return(false);
}
Logger.LogWarning("Failed to create audit record for delete");
return(false);
}
Logger.LogTrace("Data is not audited");
if (await DataStreamer.Delete <TDataType>(id))
{
Logger.LogInformation("Deleted data");
if (await IndexWorker.Unindex(data))
{
Logger.LogInformation("Removed any indexes for data");
return(true);
}
Logger.LogWarning("Failed to remove indexes for data");
await DataStreamer.Rollback(id, data);
Logger.LogInformation("Rolled data back");
}
Logger.LogWarning("Failed to delete data");
return(false);
}
}
public async Task <bool> Write <TDataType>(TDataType data) where TDataType : DataEntity
{
using (Logger.BeginScope("{Operation} is {Action} {DataType} with Id ({Id})", nameof(DataManager),
"writing", typeof(TDataType).Name, data.Id))
{
Logger.LogTrace("Checking if data exists");
var currentData = await Read <TDataType>(data.Id);
Logger.LogInformation(currentData is null ? "Creating data" : "Updating data");
if (typeof(TDataType).IsDefined(typeof(AuditAttribute), true))
{
Logger.LogTrace("{DataType} is audited", typeof(TDataType).Name);
var writtenData = await DataStreamer.Write(data.Id, data);
if (writtenData)
{
Logger.LogTrace("Data ready to commit to store");
var auditResult = currentData is null
? await AuditWorker.CreationEvent(data)
: await AuditWorker.UpdateEvent(data, currentData);
if (auditResult)
{
Logger.LogTrace("Audit record created");
writtenData = await DataStreamer.Commit <TDataType>(data.Id);
if (writtenData)
{
Logger.LogTrace("Data saved to store");
var committedAudit = await AuditWorker.CommitEvents(data);
if (committedAudit)
{
Logger.LogTrace("Audit saved to store");
var indexed = await IndexWorker.Index(data);
if (indexed)
{
Logger.LogInformation("Data, Audit saved successfully");
return(true);
}
Logger.LogWarning("Failed to index data");
await AuditWorker.RollbackEvent(data);
Logger.LogInformation("Added rollback to audit");
Logger.LogTrace("Rolled back data");
}
await DataStreamer.Rollback(data.Id, currentData);
Logger.LogInformation("Rolled data back");
return(false);
}
await AuditWorker.DiscardEvents(data);
return(false);
}
Logger.LogWarning("Failed to create audit record for data");
await DataStreamer.DiscardChanges <TDataType>(data.Id);
Logger.LogTrace("Discard changes");
}
Logger.LogWarning("Failed to save data to the store");
return(false);
}
Logger.LogTrace("Data is not audited");
var writeAndCommit = await DataStreamer.WriteAndCommit(data.Id, data);
if (writeAndCommit)
{
var indexed = await IndexWorker.Index(data);
if (!indexed)
{
Logger.LogWarning("Failed to index data");
await DataStreamer.Rollback(data.Id, currentData);
Logger.LogInformation("Rolled data back");
return(false);
}
Logger.LogTrace("Data was indexed");
Logger.LogInformation("Data saved successfully");
}
else
{
Logger.LogWarning("Failed to save data");
}
return(writeAndCommit);
}
}
