protected virtual Directory CreateIndex() { var index = new RAMDirectory(); var indexWriterAnalyzer = CreateAnalyzer(); var config = IndexUtils.CreateWriterConfig(indexWriterAnalyzer); using (var writer = new IndexWriter(index, config)) { void indexDocumentGroup(IEnumerable <Document> documents) { foreach (var doc in documents) { writer.AddDocument(doc); } Interlocked.Increment(ref GroupsAddedToIndex); IndexingProgress?.Invoke(); } IndexUtils.ForEach(GetDocumentGroupsToIndex(), indexDocumentGroup); writer.Flush(triggerMerge: true, applyAllDeletes: false); writer.Commit(); } return(index); }
protected virtual Directory CreateIndex(LuceneSearcherState <TId, TDoc> searcherState) { var spellchecker = CreateSpellchecker(); var state = CreateState(searcherState, spellchecker, loaded: false); bool stateExisted = State != null; if (!stateExisted) { State = state; } void progressHandler() => IndexingProgress?.Invoke(); state.IndexingProgress += progressHandler; var result = state.CreateIndex(); state.IndexingProgress -= progressHandler; if (stateExisted) { State.Dispose(); State = state; } return(result); }
private void NotifyProgress(int oldValue, int newValue) { bool oldIsIndexing = oldValue > 0; bool newIsIndexing = newValue > 0; if (oldIsIndexing == newIsIndexing) { return; } IndexingProgress?.Invoke(this, new IndexingEventArgs(newIsIndexing)); }
protected virtual Directory CreateIndex(LuceneSearcherState <TId, TDoc> state) { void progressHandler() => IndexingProgress?.Invoke(); state.IndexingProgress += progressHandler; var result = state.CreateIndex(); state.IndexingProgress -= progressHandler; return(result); }
protected virtual void IndexField(KeyValuePair <string, HashSet <string> > pair) { var words = pair.Value.OrderBy(Str.Comparer).ToReadOnlyList(); foreach (string word in words) { Spellchecker.IndexWord(pair.Key, word); } Interlocked.Increment(ref _indexedFields); IndexingProgress?.Invoke(); }
public Directory CreateIndex() { if (IsLoading || IsLoaded) { throw new InvalidOperationException(); } IsLoading = true; var index = new RAMDirectory(); var indexWriterAnalyzer = Adapter.CreateAnalyzer(); var config = IndexUtils.CreateWriterConfig(indexWriterAnalyzer); using (var writer = new IndexWriter(index, config)) { void indexDocumentGroup(IEnumerable <Document> documents) { if (_aborted) { return; } foreach (var doc in documents) { if (_aborted) { return; } writer.AddDocument(doc); } Interlocked.Increment(ref GroupsAddedToIndex); IndexingProgress?.Invoke(); } IndexUtils.ForEach(GetDocumentGroupsToIndex(), indexDocumentGroup); if (_aborted) { return(null); } writer.Flush(triggerMerge: true, applyAllDeletes: false); writer.Commit(); } IsLoading = false; return(index); }
private void indexField(KeyValuePair <string, HashSet <string> > pair) { if (_abort) { return; } var words = pair.Value.OrderBy(Str.Comparer).ToReadOnlyList(); foreach (string word in words) { if (_abort) { return; } _spellchecker.IndexWord(pair.Key, word); } Interlocked.Increment(ref _indexedFields); IndexingProgress?.Invoke(); }
public void Progress(IndexingProgress progress) { _log.LogTrace(progress.Description); _totalCountMap[progress.DocumentType] = progress.TotalCount ?? 0; _processedCountMap[progress.DocumentType] = progress.ProcessedCount ?? 0; _notification.DocumentType = progress.DocumentType; _notification.Description = progress.Description; if (!progress.Errors.IsNullOrEmpty()) { _notification.Errors.AddRange(progress.Errors); _notification.ErrorCount = _notification.Errors.Count; } _notification.TotalCount = progress.TotalCount ?? 0; _notification.ProcessedCount = progress.ProcessedCount ?? 0; if (!_suppressInsignificantNotifications || progress.TotalCount > 0 || progress.ProcessedCount > 0) { _pushNotificationManager.Send(_notification); } }
public Directory CreateIndex() { if (IsLoaded) { throw new InvalidOperationException(); } IsLoading = true; var index = new RAMDirectory(); IReadOnlyList <(string UserField, string Language)> tasks = _adapter.GetUserFields() .Where(_adapter.IsIndexedInSpellchecker) .SelectMany(f => _adapter.GetFieldLanguages(f).Select(l => (f, l))) .ToReadOnlyList(); TotalFields = tasks.Count; var indexedWordsByField = new Dictionary <string, HashSet <string> >(Str.Comparer); void getContentToIndex((string UserField, string Language) task) { var values = _adapter.IsStoredInSpellchecker(task.UserField, task.Language) ? GetObjectsToIndex().SelectMany(c => _adapter.GetSpellcheckerValues(c, task.UserField, task.Language)) : GetValuesCache(task.UserField, task.Language); var spellcheckerField = _adapter.GetSpellcheckerFieldIn(task.UserField, task.Language); HashSet <string> indexedWords; lock (indexedWordsByField) { if (!indexedWordsByField.TryGetValue(spellcheckerField, out indexedWords)) { indexedWords = new HashSet <string>(Str.Comparer); indexedWordsByField.Add(spellcheckerField, indexedWords); } } lock (indexedWords) foreach (string value in values) { indexedWords.Add(value); } } IndexUtils.ForEach(tasks, getContentToIndex); TotalFields = indexedWordsByField.Count; _indexedFields = 0; _spellchecker.BeginIndex(index); IndexUtils.ForEach( indexedWordsByField, indexField, suppressParallelism: true); if (_abort) { return(null); } _spellchecker.EndIndex(); IsLoading = false; IndexingProgress?.Invoke(); IsLoaded = true; return(index); }
protected void InvokeIndexingProgressEvent() => IndexingProgress?.Invoke();