/// <summary> /// Removes all linkage from the OMNI Framework for the passed in Assembly. /// </summary> /// <param name="iResult"></param> internal void DeIndexClass(IndexingResult iResult) { if (iResult == null) { return; } lock (tick) { foreach (string c in iResult.ClassesFound) { for (int i = 0; i < SimDictSize; i++) { if (_msimdict[i] == null) { continue; } if (_msimdict[i].GetType().FullName.ToLower() != c) { continue; } self.Error(">>csFactory::INFO: Deleting Proxy Object for class '" + _msimdict[i].GetType().FullName.ToLower() + "'. ID: " + _msimdict[i].GetID() + " Name: " + _msimdict[i].getName()); _msimdict[i].delete(); _msimdict[i] = null; } MObjectCreateDefs.Remove(c); MCustomClassDef.Remove(c); } foreach (string g in iResult.GlobalFunctsFound) { _mGlobalFuncts.Remove(g); } } }
private static void AssertIndexActionSucceeded(string key, IndexingResult result, int expectedStatusCode) { Assert.Equal(key, result.Key); Assert.True(result.Succeeded); Assert.Null(result.ErrorMessage); Assert.Equal(expectedStatusCode, result.StatusCode); }
protected virtual async Task <IndexingResult> ProcessDocumentsAsync(IndexDocumentChangeType changeType, string[] changedIds, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); IndexingResult result = null; if (changeType == IndexDocumentChangeType.Deleted) { result = await DeleteDocumentsAsync(batchOptions.DocumentType, changedIds); } else if (changeType is IndexDocumentChangeType.Modified or IndexDocumentChangeType.Created) { var documents = await GetDocumentsAsync(changedIds, batchOptions.PrimaryDocumentBuilder, batchOptions.SecondaryDocumentBuilders, cancellationToken); if (batchOptions.Reindex && _searchProvider is ISupportIndexSwap supportIndexSwapProvider) { result = await supportIndexSwapProvider.IndexWithBackupAsync(batchOptions.DocumentType, documents); } else { result = await _searchProvider.IndexAsync(batchOptions.DocumentType, documents); } } return(result); }
protected virtual async Task <IndexingResult> ProcessPartialDocumentsAsync( IEnumerable <IndexDocumentChange> changes, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken) { var result = new IndexingResult { Items = new List <IndexingResultItem>() }; var indexDocumentChanges = changes as IndexDocumentChange[] ?? changes.ToArray(); var changeIds = indexDocumentChanges.Select(x => x.DocumentId).Distinct(); foreach (var id in changeIds) { var builders = indexDocumentChanges .Where(x => x.DocumentId == id) .SelectMany(x => _configs.GetBuildersForProvider(x.Provider.GetType())); var documents = await GetDocumentsAsync(new[] { id }, null, builders, cancellationToken); IndexingResult indexingResult; indexingResult = await((ISupportPartialUpdate)_searchProvider).IndexPartialAsync(batchOptions.DocumentType, documents); result.Items.AddRange(indexingResult.Items); } return(result); }
protected virtual async Task <IndexingResult> ProcessChangesAsync(IEnumerable <IndexDocumentChange> changes, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var result = new IndexingResult(); var groups = GetLatestChangesForEachDocumentGroupedByChangeType(changes); foreach (var group in groups) { var changeType = group.Key; var documentIds = group.Value; var groupResult = await ProcessDocumentsAsync(changeType, documentIds, batchOptions, cancellationToken); if (groupResult?.Items != null) { if (result.Items == null) { result.Items = new List <IndexingResultItem>(); } result.Items.AddRange(groupResult.Items); } } return(result); }
public virtual async Task <IndexingResult> RemoveAsync(string documentType, IList <IndexDocument> documents) { var providerDocuments = documents.Select(d => new SearchDocument { Id = d.Id }).ToArray(); var indexName = GetIndexName(documentType); var bulkDefinition = new BulkDescriptor(); bulkDefinition.DeleteMany(providerDocuments).Index(indexName).Type(documentType); var bulkResponse = await Client.BulkAsync(bulkDefinition); await Client.RefreshAsync(indexName); var result = new IndexingResult { Items = bulkResponse.Items.Select(i => new IndexingResultItem { Id = i.Id, Succeeded = i.IsValid, ErrorMessage = i.Error?.Reason, }).ToArray(), }; return(result); }
public async Task <Response <IndexingResult> > InsertAync() { // 1 - Pesquisa no controle de indexação o id da última pesquisa indexada Indexing ultimaIndexacao = await indexingRepository.SearchLastIndexedQueryAsync(); // 2 - Pesquisa no bases de pesquisas salvas todas as entradas novas ou as que sofreram alteração List <string> pesquisasDisponiveisIds = await queryQueueRepository.SearchAvaliableQueryIdsAsync(ultimaIndexacao.DataIndexado); // 3 - Havendo pesquisas disponíveis para indexação, realiza o cadastro da indexação com o status "Aguardando indexação" foreach (var item in pesquisasDisponiveisIds) { Indexing indexacao = new Indexing { PesquisaId = item, StatusId = 1, }; await indexingRepository.InsertIndexingAsync(indexacao); } await queryQueueService.InsertAync(); IndexingResult resultado = new IndexingResult { TotalIndexedDocuments = pesquisasDisponiveisIds.Count }; return(Response.CreateResponse(resultado)); }
protected virtual IList <string> GetIndexingErrors(IndexingResult indexingResult) { var errors = indexingResult?.Items? .Where(i => !i.Succeeded) .Select(i => $"ID: {i.Id}, Error: {i.ErrorMessage}") .ToList(); return(errors?.Any() == true ? errors : null); }
private static void AssertIndexActionFailed( string key, IndexingResult result, string expectedMessage, int expectedStatusCode) { Assert.Equal(key, result.Key); Assert.False(result.Succeeded); Assert.Equal(expectedMessage, result.ErrorMessage); Assert.Equal(expectedStatusCode, result.StatusCode); }
public virtual async Task <IndexingResult> IndexDocumentsAsync(string documentType, string[] documentIds, IEnumerable <string> builderTypes = null) { // Todo: reuse general index api? var configs = _configs.Where(c => c.DocumentType.EqualsInvariant(documentType)).ToArray(); var result = new IndexingResult { Items = new List <IndexingResultItem>() }; var partialUpdate = false; foreach (var config in configs) { var primaryDocumentBuilder = config.DocumentSource.DocumentBuilder; var additionalDocumentBuilders = config.RelatedSources? .Where(s => s.DocumentBuilder != null) .Select(s => s.DocumentBuilder) .ToList() ?? new List <IIndexDocumentBuilder>(); if ((builderTypes?.Any() ?? false) && additionalDocumentBuilders.Any() && _searchProvider is ISupportPartialUpdate) { additionalDocumentBuilders = additionalDocumentBuilders.Where(x => builderTypes.Contains(x.GetType().FullName)) .ToList(); // In case of changing main object itself, there would be only primary document builder, // but in the other cases, when changed additional dependent objects, primary builder should be nulled. if (!builderTypes.Contains(primaryDocumentBuilder.GetType().FullName)) { primaryDocumentBuilder = null; } partialUpdate = true; } var documents = await GetDocumentsAsync(documentIds, primaryDocumentBuilder, additionalDocumentBuilders, new CancellationTokenWrapper(CancellationToken.None)); IndexingResult indexingResult; if (partialUpdate && _searchProvider is ISupportPartialUpdate supportPartialUpdateProvider) { indexingResult = await supportPartialUpdateProvider.IndexPartialAsync(documentType, documents); } else { indexingResult = await _searchProvider.IndexAsync(documentType, documents); } result.Items.AddRange(indexingResult.Items ?? Enumerable.Empty <IndexingResultItem>()); } return(result); }
/// <summary> /// Attempt to add an item to the retry queue or raise a failure /// notification if it's been retried too many times. /// </summary> /// <param name="action">The action to retry.</param> /// <param name="result">Result of executing the action.</param> /// <param name="exception">An exception raised by the action.</param> /// <param name="cancellationToken">A cancellation token.</param> /// <returns>Task.</returns> private async Task EnqueueOrFailRetryAsync( PublisherAction <IndexDocumentsAction <T> > action, IndexingResult result, Exception exception, CancellationToken cancellationToken) { if (!EnqueueRetry(action)) { await _sender.OnActionFailedAsync( action.Document, result, exception, cancellationToken) .ConfigureAwait(false); } }
public virtual Task <IndexingResult> RemoveAsync(string documentType, IList <IndexDocument> documents) { var result = new IndexingResult { Items = new List <IndexingResultItem>(documents.Count) }; var indexName = GetIndexName(documentType); var directoryPath = GetDirectoryPath(indexName); CloseWriter(indexName, false); Analyzer analyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48); IndexWriterConfig config = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer); using (var directory = FSDirectory.Open(directoryPath)) using (var reader = new IndexWriter(directory, config)) { foreach (var document in documents) { var resultItem = new IndexingResultItem { Id = document.Id }; result.Items.Add(resultItem); try { if (!string.IsNullOrEmpty(document.Id)) { var term = new Term(LuceneSearchHelper.KeyFieldName, document.Id); reader.DeleteDocuments(term); resultItem.Succeeded = true; } else { resultItem.ErrorMessage = "Document ID is empty"; } } catch (Exception ex) { resultItem.ErrorMessage = ex.ToString(); } } } return(Task.FromResult(result)); }
protected virtual async Task <IndexingResult> ProcessDocumentsAsync(IndexDocumentChangeType changeType, IList <string> documentIds, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); IndexingResult result = null; if (changeType == IndexDocumentChangeType.Deleted) { result = await DeleteDocumentsAsync(batchOptions.DocumentType, documentIds.ToArray()); } else if (changeType == IndexDocumentChangeType.Modified) { result = await IndexDocumentsAsync(batchOptions.DocumentType, documentIds, batchOptions.PrimaryDocumentBuilder, batchOptions.SecondaryDocumentBuilders, cancellationToken); } return(result); }
public virtual Task <IndexingResult> IndexAsync(string documentType, IList <IndexDocument> documents) { var result = new IndexingResult { Items = new List <IndexingResultItem>(documents.Count) }; var indexName = GetIndexName(documentType); lock (_providerlock) { var writer = GetIndexWriter(indexName, true, false); foreach (var document in documents) { var resultItem = new IndexingResultItem { Id = document.Id }; result.Items.Add(resultItem); try { var providerDocument = ConvertToProviderDocument(document, documentType); if (!string.IsNullOrEmpty(document.Id)) { var term = new Term(LuceneSearchHelper.KeyFieldName, document.Id); writer.UpdateDocument(term, providerDocument); resultItem.Succeeded = true; } else { resultItem.ErrorMessage = "Document ID is empty"; } } catch (Exception ex) { resultItem.ErrorMessage = ex.ToString(); } } } CloseWriter(indexName, true); return(Task.FromResult(result)); }
protected virtual async Task <IndexingResult> ProcessChangesAsync(IEnumerable <IndexDocumentChange> changes, BatchIndexingOptions batchOptions, ICancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var result = new IndexingResult { Items = new List <IndexingResultItem>() }; var indexDocumentChanges = changes as IndexDocumentChange[] ?? changes.ToArray(); // Full changes don't have changes provider specified because we don't set it for manual indexation. var fullChanges = _searchProvider is ISupportPartialUpdate?indexDocumentChanges .Where(x => x.ChangeType is IndexDocumentChangeType.Deleted or IndexDocumentChangeType.Created || !_configs.GetBuildersForProvider(x.Provider?.GetType()).Any() ) .ToArray() : indexDocumentChanges; var partialChanges = indexDocumentChanges.Except(fullChanges); var partialResult = await ProcessPartialDocumentsAsync(partialChanges, batchOptions, cancellationToken); var groups = GetLatestChangesForEachDocumentGroupedByChangeType(fullChanges); foreach (var group in groups) { var changeType = group.Key; var changesGroup = group.Value; var groupResult = await ProcessDocumentsAsync(changeType, changesGroup, batchOptions, cancellationToken); if (groupResult?.Items != null) { result.Items.AddRange(groupResult.Items); } } result.Items.AddRange(partialResult.Items); return(result); }
public virtual async Task <IndexingResult> IndexAsync(string documentType, IList <IndexDocument> documents) { var indexName = GetIndexName(documentType); var providerFields = await GetMappingAsync(indexName, documentType); var oldFieldsCount = providerFields.Count(); var providerDocuments = documents.Select(document => ConvertToProviderDocument(document, providerFields, documentType)).ToList(); var updateMapping = providerFields.Count() != oldFieldsCount; var indexExits = await IndexExistsAsync(indexName); if (!indexExits) { await CreateIndexAsync(indexName, documentType); } if (!indexExits || updateMapping) { await UpdateMappingAsync(indexName, documentType, providerFields); } var bulkDefinition = new BulkDescriptor(); bulkDefinition.IndexMany(providerDocuments).Index(indexName).Type(documentType); var bulkResponse = await Client.BulkAsync(bulkDefinition); await Client.RefreshAsync(indexName); var result = new IndexingResult { Items = bulkResponse.Items.Select(i => new IndexingResultItem { Id = i.Id, Succeeded = i.IsValid, ErrorMessage = i.Error?.Reason, }).ToArray(), }; return(result); }
public virtual async Task <IndexingResult> IndexDocumentsAsync(string documentType, string[] documentIds) { // Todo: reuse general index api? var configs = _configs.Where(c => c.DocumentType.EqualsInvariant(documentType)).ToArray(); var result = new IndexingResult { Items = new List <IndexingResultItem>() }; foreach (var config in configs) { var secondaryDocBuilders = config.RelatedSources? .Where(s => s.DocumentBuilder != null) .Select(s => s.DocumentBuilder) .ToList(); var configResult = await IndexDocumentsAsync(documentType, documentIds, config.DocumentSource.DocumentBuilder, secondaryDocBuilders, new CancellationTokenWrapper(CancellationToken.None)); result.Items.AddRange(configResult.Items ?? Enumerable.Empty <IndexingResultItem>()); } return(result); }
/// <summary> /// This function reads the passed in Assembly and parses all of the Omni Decorations out and makes them available in the engine. /// Used mainly for internal purposes. /// </summary> /// <param name="mAssembly"></param> /// <returns></returns> internal IndexingResult IndexAssembly(Assembly mAssembly) { IndexingResult iResult = new IndexingResult(); List <Type> classlist = mAssembly.GetTypes().ToList(); classlist.Sort((x, y) => String.CompareOrdinal(x.FullName, y.FullName)); List <CustomClassFunctDef> staticfunctions = new List <CustomClassFunctDef>(); foreach (Type c in classlist) { if (c.FullName != null && c.FullName.StartsWith(_mNameSpace + ".Models.User")) { CustomClassDef ccd = new CustomClassDef { mType = c, mClassName = c.FullName.ToLower(), mCtor = c.GetConstructor(new Type[] {}) }; #region Constructor check #if DEBUG ConstructorInfo ci; try { ci = c.GetConstructor(new Type[] {}); } catch (Exception) { throw new Exception("Class " + c.FullName + " does not have a empty constructor."); } ccd.mCtor = ci; #endif #endregion List <MethodInfo> miList = c.GetMethods(BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance).ToList(); miList.Sort((x, y) => String.CompareOrdinal(x.Name, y.Name)); foreach (MethodInfo mi in miList) { #region Process MethodInfo if (mi.IsStatic && !mi.IsSpecialName && isExposedFunction(mi)) { if (getAlias(mi) != "") { CustomClassFunctDef ccfd = new CustomClassFunctDef { mFunctName = getAlias(mi).ToLower(), mFunctPtr = mi }; staticfunctions.Add(ccfd); } else { CustomClassFunctDef ccfd = new CustomClassFunctDef { mFunctName = mi.Name.ToLower(), mFunctPtr = mi }; staticfunctions.Add(ccfd); } } else if (!mi.IsSpecialName && isExposedFunction(mi)) { if ((mi.GetBaseDefinition().DeclaringType != mi.DeclaringType) || (mi.GetBaseDefinition().DeclaringType.Namespace.StartsWith(_mNameSpace + ".Models.User"))) { if (getAlias(mi) != "") { CustomClassFunctDef ccfd = new CustomClassFunctDef { mFunctName = getAlias(mi).ToLower(), mFunctPtr = mi }; ccd.mFuncts.Add(getAlias(mi), ccfd); } else { CustomClassFunctDef ccfd = new CustomClassFunctDef { mFunctName = mi.Name.ToLower(), mFunctPtr = mi }; ccd.mFuncts.Add(ccfd.mFunctName, ccfd); } } } else if (!mi.IsSpecialName && isExposedSystemFunction(mi)) { if (!ccd.Allowedcallbacks.Contains(mi.Name.ToLower())) { ccd.Allowedcallbacks.Add(mi.Name.ToLower()); } } #endregion } //Don't add any empty classes to the dictionary. if (!ccd.mFuncts.Any() && !ccd.Allowedcallbacks.Any()) { continue; } MCustomClassDef.Add(ccd.mClassName, ccd); iResult.ClassesFound.Add(ccd.mClassName); } #if DEBUG else if (!c.FullName.StartsWith(_mNameSpace + ".Models.Base")) { Console.WriteLine("!!!WARNING!!!! Skipping C# Type : '" + c.FullName + "'."); } #endif } //}); //We are going to sort them to improve performance. staticfunctions.Sort((x, y) => String.CompareOrdinal(x.mFunctName, y.mFunctName)); foreach (CustomClassFunctDef ccfd in staticfunctions) { _mGlobalFuncts.Add(ccfd.mFunctName, ccfd); iResult.GlobalFunctsFound.Add(ccfd.mFunctName); } return(iResult); }