/// <summary>
/// Removes all linkage from the OMNI Framework for the passed in Assembly.
/// </summary>
/// <param name="iResult"></param>
internal void DeIndexClass(IndexingResult iResult)
{
if (iResult == null)
{
return;
}
lock (tick)
{
foreach (string c in iResult.ClassesFound)
{
for (int i = 0; i < SimDictSize; i++)
{
if (_msimdict[i] == null)
{
continue;
}
if (_msimdict[i].GetType().FullName.ToLower() != c)
{
continue;
}
self.Error(">>csFactory::INFO: Deleting Proxy Object for class '" + _msimdict[i].GetType().FullName.ToLower() + "'. ID: " + _msimdict[i].GetID() + " Name: " + _msimdict[i].getName());
_msimdict[i].delete();
_msimdict[i] = null;
}
MObjectCreateDefs.Remove(c);
MCustomClassDef.Remove(c);
}
foreach (string g in iResult.GlobalFunctsFound)
{
_mGlobalFuncts.Remove(g);
}
}
}
private static void AssertIndexActionSucceeded(string key, IndexingResult result, int expectedStatusCode)
{
Assert.Equal(key, result.Key);
Assert.True(result.Succeeded);
Assert.Null(result.ErrorMessage);
Assert.Equal(expectedStatusCode, result.StatusCode);
}
protected virtual async Task <IndexingResult> ProcessDocumentsAsync(IndexDocumentChangeType changeType,
string[] changedIds, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
IndexingResult result = null;
if (changeType == IndexDocumentChangeType.Deleted)
{
result = await DeleteDocumentsAsync(batchOptions.DocumentType, changedIds);
}
else if (changeType is IndexDocumentChangeType.Modified or IndexDocumentChangeType.Created)
{
var documents = await GetDocumentsAsync(changedIds, batchOptions.PrimaryDocumentBuilder, batchOptions.SecondaryDocumentBuilders, cancellationToken);
if (batchOptions.Reindex && _searchProvider is ISupportIndexSwap supportIndexSwapProvider)
{
result = await supportIndexSwapProvider.IndexWithBackupAsync(batchOptions.DocumentType, documents);
}
else
{
result = await _searchProvider.IndexAsync(batchOptions.DocumentType, documents);
}
}
return(result);
}
protected virtual async Task <IndexingResult> ProcessPartialDocumentsAsync(
IEnumerable <IndexDocumentChange> changes,
BatchIndexingOptions batchOptions,
ICancellationToken cancellationToken)
{
var result = new IndexingResult {
Items = new List <IndexingResultItem>()
};
var indexDocumentChanges = changes as IndexDocumentChange[] ?? changes.ToArray();
var changeIds = indexDocumentChanges.Select(x => x.DocumentId).Distinct();
foreach (var id in changeIds)
{
var builders = indexDocumentChanges
.Where(x => x.DocumentId == id)
.SelectMany(x => _configs.GetBuildersForProvider(x.Provider.GetType()));
var documents = await GetDocumentsAsync(new[] { id }, null, builders, cancellationToken);
IndexingResult indexingResult;
indexingResult = await((ISupportPartialUpdate)_searchProvider).IndexPartialAsync(batchOptions.DocumentType, documents);
result.Items.AddRange(indexingResult.Items);
}
return(result);
}
protected virtual async Task <IndexingResult> ProcessChangesAsync(IEnumerable <IndexDocumentChange> changes, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var result = new IndexingResult();
var groups = GetLatestChangesForEachDocumentGroupedByChangeType(changes);
foreach (var group in groups)
{
var changeType = group.Key;
var documentIds = group.Value;
var groupResult = await ProcessDocumentsAsync(changeType, documentIds, batchOptions, cancellationToken);
if (groupResult?.Items != null)
{
if (result.Items == null)
{
result.Items = new List <IndexingResultItem>();
}
result.Items.AddRange(groupResult.Items);
}
}
return(result);
}
public virtual async Task <IndexingResult> RemoveAsync(string documentType, IList <IndexDocument> documents)
{
var providerDocuments = documents.Select(d => new SearchDocument {
Id = d.Id
}).ToArray();
var indexName = GetIndexName(documentType);
var bulkDefinition = new BulkDescriptor();
bulkDefinition.DeleteMany(providerDocuments).Index(indexName).Type(documentType);
var bulkResponse = await Client.BulkAsync(bulkDefinition);
await Client.RefreshAsync(indexName);
var result = new IndexingResult
{
Items = bulkResponse.Items.Select(i => new IndexingResultItem
{
Id = i.Id,
Succeeded = i.IsValid,
ErrorMessage = i.Error?.Reason,
}).ToArray(),
};
return(result);
}
public async Task <Response <IndexingResult> > InsertAync()
{
// 1 - Pesquisa no controle de indexação o id da última pesquisa indexada
Indexing ultimaIndexacao = await indexingRepository.SearchLastIndexedQueryAsync();
// 2 - Pesquisa no bases de pesquisas salvas todas as entradas novas ou as que sofreram alteração
List <string> pesquisasDisponiveisIds = await queryQueueRepository.SearchAvaliableQueryIdsAsync(ultimaIndexacao.DataIndexado);
// 3 - Havendo pesquisas disponíveis para indexação, realiza o cadastro da indexação com o status "Aguardando indexação"
foreach (var item in pesquisasDisponiveisIds)
{
Indexing indexacao = new Indexing
{
PesquisaId = item,
StatusId = 1,
};
await indexingRepository.InsertIndexingAsync(indexacao);
}
await queryQueueService.InsertAync();
IndexingResult resultado = new IndexingResult {
TotalIndexedDocuments = pesquisasDisponiveisIds.Count
};
return(Response.CreateResponse(resultado));
}
protected virtual IList <string> GetIndexingErrors(IndexingResult indexingResult)
{
var errors = indexingResult?.Items?
.Where(i => !i.Succeeded)
.Select(i => $"ID: {i.Id}, Error: {i.ErrorMessage}")
.ToList();
return(errors?.Any() == true ? errors : null);
}
private static void AssertIndexActionFailed(
string key,
IndexingResult result,
string expectedMessage,
int expectedStatusCode)
{
Assert.Equal(key, result.Key);
Assert.False(result.Succeeded);
Assert.Equal(expectedMessage, result.ErrorMessage);
Assert.Equal(expectedStatusCode, result.StatusCode);
}
public virtual async Task <IndexingResult> IndexDocumentsAsync(string documentType, string[] documentIds, IEnumerable <string> builderTypes = null)
{
// Todo: reuse general index api?
var configs = _configs.Where(c => c.DocumentType.EqualsInvariant(documentType)).ToArray();
var result = new IndexingResult {
Items = new List <IndexingResultItem>()
};
var partialUpdate = false;
foreach (var config in configs)
{
var primaryDocumentBuilder = config.DocumentSource.DocumentBuilder;
var additionalDocumentBuilders = config.RelatedSources?
.Where(s => s.DocumentBuilder != null)
.Select(s => s.DocumentBuilder)
.ToList() ?? new List <IIndexDocumentBuilder>();
if ((builderTypes?.Any() ?? false) && additionalDocumentBuilders.Any() && _searchProvider is ISupportPartialUpdate)
{
additionalDocumentBuilders = additionalDocumentBuilders.Where(x => builderTypes.Contains(x.GetType().FullName))
.ToList();
// In case of changing main object itself, there would be only primary document builder,
// but in the other cases, when changed additional dependent objects, primary builder should be nulled.
if (!builderTypes.Contains(primaryDocumentBuilder.GetType().FullName))
{
primaryDocumentBuilder = null;
}
partialUpdate = true;
}
var documents = await GetDocumentsAsync(documentIds, primaryDocumentBuilder, additionalDocumentBuilders, new CancellationTokenWrapper(CancellationToken.None));
IndexingResult indexingResult;
if (partialUpdate && _searchProvider is ISupportPartialUpdate supportPartialUpdateProvider)
{
indexingResult = await supportPartialUpdateProvider.IndexPartialAsync(documentType, documents);
}
else
{
indexingResult = await _searchProvider.IndexAsync(documentType, documents);
}
result.Items.AddRange(indexingResult.Items ?? Enumerable.Empty <IndexingResultItem>());
}
return(result);
}
/// <summary>
/// Attempt to add an item to the retry queue or raise a failure
/// notification if it's been retried too many times.
/// </summary>
/// <param name="action">The action to retry.</param>
/// <param name="result">Result of executing the action.</param>
/// <param name="exception">An exception raised by the action.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>Task.</returns>
private async Task EnqueueOrFailRetryAsync(
PublisherAction <IndexDocumentsAction <T> > action,
IndexingResult result,
Exception exception,
CancellationToken cancellationToken)
{
if (!EnqueueRetry(action))
{
await _sender.OnActionFailedAsync(
action.Document,
result,
exception,
cancellationToken)
.ConfigureAwait(false);
}
}
public virtual Task <IndexingResult> RemoveAsync(string documentType, IList <IndexDocument> documents)
{
var result = new IndexingResult
{
Items = new List <IndexingResultItem>(documents.Count)
};
var indexName = GetIndexName(documentType);
var directoryPath = GetDirectoryPath(indexName);
CloseWriter(indexName, false);
Analyzer analyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48);
IndexWriterConfig config = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer);
using (var directory = FSDirectory.Open(directoryPath))
using (var reader = new IndexWriter(directory, config))
{
foreach (var document in documents)
{
var resultItem = new IndexingResultItem {
Id = document.Id
};
result.Items.Add(resultItem);
try
{
if (!string.IsNullOrEmpty(document.Id))
{
var term = new Term(LuceneSearchHelper.KeyFieldName, document.Id);
reader.DeleteDocuments(term);
resultItem.Succeeded = true;
}
else
{
resultItem.ErrorMessage = "Document ID is empty";
}
}
catch (Exception ex)
{
resultItem.ErrorMessage = ex.ToString();
}
}
}
return(Task.FromResult(result));
}
protected virtual async Task <IndexingResult> ProcessDocumentsAsync(IndexDocumentChangeType changeType, IList <string> documentIds, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
IndexingResult result = null;
if (changeType == IndexDocumentChangeType.Deleted)
{
result = await DeleteDocumentsAsync(batchOptions.DocumentType, documentIds.ToArray());
}
else if (changeType == IndexDocumentChangeType.Modified)
{
result = await IndexDocumentsAsync(batchOptions.DocumentType, documentIds, batchOptions.PrimaryDocumentBuilder, batchOptions.SecondaryDocumentBuilders, cancellationToken);
}
return(result);
}
public virtual Task <IndexingResult> IndexAsync(string documentType, IList <IndexDocument> documents)
{
var result = new IndexingResult
{
Items = new List <IndexingResultItem>(documents.Count)
};
var indexName = GetIndexName(documentType);
lock (_providerlock)
{
var writer = GetIndexWriter(indexName, true, false);
foreach (var document in documents)
{
var resultItem = new IndexingResultItem {
Id = document.Id
};
result.Items.Add(resultItem);
try
{
var providerDocument = ConvertToProviderDocument(document, documentType);
if (!string.IsNullOrEmpty(document.Id))
{
var term = new Term(LuceneSearchHelper.KeyFieldName, document.Id);
writer.UpdateDocument(term, providerDocument);
resultItem.Succeeded = true;
}
else
{
resultItem.ErrorMessage = "Document ID is empty";
}
}
catch (Exception ex)
{
resultItem.ErrorMessage = ex.ToString();
}
}
}
CloseWriter(indexName, true);
return(Task.FromResult(result));
}
protected virtual async Task <IndexingResult> ProcessChangesAsync(IEnumerable <IndexDocumentChange> changes,
BatchIndexingOptions batchOptions, ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var result = new IndexingResult {
Items = new List <IndexingResultItem>()
};
var indexDocumentChanges = changes as IndexDocumentChange[] ?? changes.ToArray();
// Full changes don't have changes provider specified because we don't set it for manual indexation.
var fullChanges = _searchProvider is ISupportPartialUpdate?indexDocumentChanges
.Where(x =>
x.ChangeType is IndexDocumentChangeType.Deleted or IndexDocumentChangeType.Created ||
!_configs.GetBuildersForProvider(x.Provider?.GetType()).Any()
)
.ToArray() : indexDocumentChanges;
var partialChanges = indexDocumentChanges.Except(fullChanges);
var partialResult = await ProcessPartialDocumentsAsync(partialChanges, batchOptions, cancellationToken);
var groups = GetLatestChangesForEachDocumentGroupedByChangeType(fullChanges);
foreach (var group in groups)
{
var changeType = group.Key;
var changesGroup = group.Value;
var groupResult =
await ProcessDocumentsAsync(changeType, changesGroup, batchOptions, cancellationToken);
if (groupResult?.Items != null)
{
result.Items.AddRange(groupResult.Items);
}
}
result.Items.AddRange(partialResult.Items);
return(result);
}
public virtual async Task <IndexingResult> IndexAsync(string documentType, IList <IndexDocument> documents)
{
var indexName = GetIndexName(documentType);
var providerFields = await GetMappingAsync(indexName, documentType);
var oldFieldsCount = providerFields.Count();
var providerDocuments = documents.Select(document => ConvertToProviderDocument(document, providerFields, documentType)).ToList();
var updateMapping = providerFields.Count() != oldFieldsCount;
var indexExits = await IndexExistsAsync(indexName);
if (!indexExits)
{
await CreateIndexAsync(indexName, documentType);
}
if (!indexExits || updateMapping)
{
await UpdateMappingAsync(indexName, documentType, providerFields);
}
var bulkDefinition = new BulkDescriptor();
bulkDefinition.IndexMany(providerDocuments).Index(indexName).Type(documentType);
var bulkResponse = await Client.BulkAsync(bulkDefinition);
await Client.RefreshAsync(indexName);
var result = new IndexingResult
{
Items = bulkResponse.Items.Select(i => new IndexingResultItem
{
Id = i.Id,
Succeeded = i.IsValid,
ErrorMessage = i.Error?.Reason,
}).ToArray(),
};
return(result);
}
public virtual async Task <IndexingResult> IndexDocumentsAsync(string documentType, string[] documentIds)
{
// Todo: reuse general index api?
var configs = _configs.Where(c => c.DocumentType.EqualsInvariant(documentType)).ToArray();
var result = new IndexingResult {
Items = new List <IndexingResultItem>()
};
foreach (var config in configs)
{
var secondaryDocBuilders = config.RelatedSources?
.Where(s => s.DocumentBuilder != null)
.Select(s => s.DocumentBuilder)
.ToList();
var configResult = await IndexDocumentsAsync(documentType, documentIds,
config.DocumentSource.DocumentBuilder, secondaryDocBuilders,
new CancellationTokenWrapper(CancellationToken.None));
result.Items.AddRange(configResult.Items ?? Enumerable.Empty <IndexingResultItem>());
}
return(result);
}
/// <summary>
/// This function reads the passed in Assembly and parses all of the Omni Decorations out and makes them available in the engine.
/// Used mainly for internal purposes.
/// </summary>
/// <param name="mAssembly"></param>
/// <returns></returns>
internal IndexingResult IndexAssembly(Assembly mAssembly)
{
IndexingResult iResult = new IndexingResult();
List <Type> classlist = mAssembly.GetTypes().ToList();
classlist.Sort((x, y) => String.CompareOrdinal(x.FullName, y.FullName));
List <CustomClassFunctDef> staticfunctions = new List <CustomClassFunctDef>();
foreach (Type c in classlist)
{
if (c.FullName != null && c.FullName.StartsWith(_mNameSpace + ".Models.User"))
{
CustomClassDef ccd = new CustomClassDef {
mType = c, mClassName = c.FullName.ToLower(), mCtor = c.GetConstructor(new Type[] {})
};
#region Constructor check
#if DEBUG
ConstructorInfo ci;
try
{
ci = c.GetConstructor(new Type[] {});
}
catch (Exception)
{
throw new Exception("Class " + c.FullName + " does not have a empty constructor.");
}
ccd.mCtor = ci;
#endif
#endregion
List <MethodInfo> miList = c.GetMethods(BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance).ToList();
miList.Sort((x, y) => String.CompareOrdinal(x.Name, y.Name));
foreach (MethodInfo mi in miList)
{
#region Process MethodInfo
if (mi.IsStatic && !mi.IsSpecialName && isExposedFunction(mi))
{
if (getAlias(mi) != "")
{
CustomClassFunctDef ccfd = new CustomClassFunctDef {
mFunctName = getAlias(mi).ToLower(), mFunctPtr = mi
};
staticfunctions.Add(ccfd);
}
else
{
CustomClassFunctDef ccfd = new CustomClassFunctDef {
mFunctName = mi.Name.ToLower(), mFunctPtr = mi
};
staticfunctions.Add(ccfd);
}
}
else if (!mi.IsSpecialName && isExposedFunction(mi))
{
if ((mi.GetBaseDefinition().DeclaringType != mi.DeclaringType) || (mi.GetBaseDefinition().DeclaringType.Namespace.StartsWith(_mNameSpace + ".Models.User")))
{
if (getAlias(mi) != "")
{
CustomClassFunctDef ccfd = new CustomClassFunctDef {
mFunctName = getAlias(mi).ToLower(), mFunctPtr = mi
};
ccd.mFuncts.Add(getAlias(mi), ccfd);
}
else
{
CustomClassFunctDef ccfd = new CustomClassFunctDef {
mFunctName = mi.Name.ToLower(), mFunctPtr = mi
};
ccd.mFuncts.Add(ccfd.mFunctName, ccfd);
}
}
}
else if (!mi.IsSpecialName && isExposedSystemFunction(mi))
{
if (!ccd.Allowedcallbacks.Contains(mi.Name.ToLower()))
{
ccd.Allowedcallbacks.Add(mi.Name.ToLower());
}
}
#endregion
}
//Don't add any empty classes to the dictionary.
if (!ccd.mFuncts.Any() && !ccd.Allowedcallbacks.Any())
{
continue;
}
MCustomClassDef.Add(ccd.mClassName, ccd);
iResult.ClassesFound.Add(ccd.mClassName);
}
#if DEBUG
else if (!c.FullName.StartsWith(_mNameSpace + ".Models.Base"))
{
Console.WriteLine("!!!WARNING!!!! Skipping C# Type : '" + c.FullName + "'.");
}
#endif
}
//});
//We are going to sort them to improve performance.
staticfunctions.Sort((x, y) => String.CompareOrdinal(x.mFunctName, y.mFunctName));
foreach (CustomClassFunctDef ccfd in staticfunctions)
{
_mGlobalFuncts.Add(ccfd.mFunctName, ccfd);
iResult.GlobalFunctsFound.Add(ccfd.mFunctName);
}
return(iResult);
}
