public AddError ( string index, string key, string error ) : void | ||
index | string | |
key | string | |
error | string | |
return | void |
public override void Remove(string[] keys, WorkContext context) { Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), name)); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); keys.Apply( key => batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, key), exception); context.AddError(name, key, exception.Message); }, trigger => trigger.OnIndexEntryDeleted(key))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, batcher => batcher.Dispose()); return(keys.Length); }); }
public override void Remove(string[] keys, WorkContext context) { Write(context, (writer, analyzer) => { if (logIndexing.IsDebugEnabled) { logIndexing.DebugFormat("Deleting ({0}) from {1}", string.Format(", ", keys), name); } var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); keys.Apply( key => batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, key); context.AddError(name, key, exception.Message); }, trigger => trigger.OnIndexEntryDeleted(name, key))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k)).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, batcher => batcher.Dispose()); return(true); }); }
protected void Write(WorkContext context, Func <IndexWriter, Analyzer, bool> action) { if (disposed) { throw new ObjectDisposedException("Index " + name + " has been disposed"); } lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List <Action>(); Analyzer analyzer = null; try { try { analyzer = CreateAnalyzer(new LowerCaseAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = new IndexWriter(directory, new StopAnalyzer(Version.LUCENE_29), IndexWriter.MaxFieldLength.UNLIMITED); } try { shouldRecreateSearcher = action(indexWriter, analyzer); foreach (var indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocumented); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } } finally { currentlyIndexDocumented.Clear(); if (analyzer != null) { analyzer.Close(); } foreach (var dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) { RecreateSearcher(); } } }
public override void Remove(string[] keys, WorkContext context) { Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), indexId)); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); keys.Apply( key => InvokeOnIndexEntryDeletedOnAllBatchers(batchers, new Term(Constants.DocumentIdFieldName, key))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(indexId, null, e.Message, "Dispose Trigger"); }, batcher => batcher.Dispose()); IndexStats currentIndexStats = null; context.TransactionalStorage.Batch(accessor => currentIndexStats = accessor.Indexing.GetIndexStats(indexId)); return(new IndexedItemsInfo { ChangedDocs = keys.Length, HighestETag = currentIndexStats.LastIndexedEtag, DeletedKeys = keys }); }); }
public override void Remove(string[] keys, WorkContext context) { Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; if (logIndexing.IsDebugEnabled) { logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), PublicName)); } var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); keys.Apply( key => InvokeOnIndexEntryDeletedOnAllBatchers(batchers, new Term(Constants.DocumentIdFieldName, key.ToLowerInvariant()))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger in " + PublicName, e); context.AddError(indexId, PublicName, null, e, "Dispose Trigger"); }, batcher => batcher.Dispose()); return(new IndexedItemsInfo(GetLastEtagFromStats()) { ChangedDocs = keys.Length, DeletedKeys = keys }); }); }
protected IEnumerable <object> RobustEnumerationReduce(IEnumerable <object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { // not strictly accurate, but if we get that many errors, probably an error anyway. return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = actions.Indexing.IncrementReduceIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementReduceIndexingAttempt, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); try { actions.Indexing.IncrementReduceIndexingFailure(); } catch (Exception e) { // we don't care about error here, because it is an error on error problem logIndexing.WarnException( String.Format("Could not increment indexing failure rate for {0}", name), e); } } }.RobustEnumeration(input, func)); }
// we don't use the usual GroupBy, because that isn't streaming // we rely on the fact that all values from the same docs are always outputed at // the same time, so we can take advantage of this fact private IEnumerable <IGrouping <object, dynamic> > GroupByDocumentId(WorkContext context, IEnumerable <object> docs) { var enumerator = docs.GetEnumerator(); if (enumerator.MoveNext() == false) { yield break; } while (true) { object documentId; try { documentId = GetDocumentId(enumerator.Current); } catch (Exception e) { context.AddError(name, null, e.Message); if (enumerator.MoveNext() == false) { yield break; } continue; } var groupByDocumentId = new Grouping(documentId, enumerator); yield return(groupByDocumentId); if (groupByDocumentId.Done) { break; } } }
private bool?MoveNext(IEnumerator en, StatefulEnumerableWrapper <object> innerEnumerator, WorkContext context, IStorageActionsAccessor actions) { try { actions.Indexing.IncrementIndexingAttempt(); var moveNext = en.MoveNext(); if (moveNext == false) { actions.Indexing.DecrementIndexingAttempt(); } return(moveNext); } catch (Exception e) { actions.Indexing.IncrementIndexingFailure(); context.AddError(name, TryGetDocKey(innerEnumerator.Current), e.Message ); log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name, GetDocId(innerEnumerator)); } return(null); }
protected IEnumerable <object> RobustEnumerationReduce(IEnumerable <object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { return(new RobustEnumerator { BeforeMoveNext = actions.Indexing.IncrementReduceIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementReduceIndexingAttempt, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnFormat(exception, "Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)); try { actions.Indexing.IncrementReduceIndexingFailure(); } catch (Exception e) { // we don't care about error here, because it is an error on error problem logIndexing.WarnFormat(e, "Could not increment indexing failure rate for {0}", name); } } }.RobustEnumeration(input, func)); }
public override void Remove(string[] keys, WorkContext context) { context.TransactionalStorage.Batch(actions => { var reduceKeyAndBuckets = new Dictionary <ReduceKeyAndBucket, int>(); foreach (var key in keys) { context.CancellationToken.ThrowIfCancellationRequested(); actions.MapReduce.DeleteMappedResultsForDocumentId(key, indexId, reduceKeyAndBuckets); } actions.MapReduce.UpdateRemovedMapReduceStats(indexId, reduceKeyAndBuckets, context.CancellationToken); foreach (var reduceKeyAndBucket in reduceKeyAndBuckets) { context.CancellationToken.ThrowIfCancellationRequested(); actions.MapReduce.ScheduleReductions(indexId, 0, reduceKeyAndBucket.Key); } }); Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; if (logIndexing.IsDebugEnabled) { logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), PublicName)); } var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); keys.Apply( key => InvokeOnIndexEntryDeletedOnAllBatchers(batchers, new Term(Constants.ReduceKeyFieldName, key.ToLowerInvariant()))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.ReduceKeyFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger in " + PublicName, e); context.AddError(indexId, PublicName, null, e, "Dispose Trigger"); }, batcher => batcher.Dispose()); return(new IndexedItemsInfo(null) { ChangedDocs = keys.Length }); }); }
// we don't care about tracking map/reduce stats here, since it is merely // an optimization step protected IEnumerable <object> RobustEnumerationReduceDuringMapPhase(IEnumerable <object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { // not strictly accurate, but if we get that many errors, probably an error anyway. return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = () => { }, // don't care CancelMoveNext = () => { }, // don't care OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); } }.RobustEnumeration(input, func)); }
protected IEnumerable <object> RobustEnumerationIndex(IEnumerable <object> input, IEnumerable <IndexingFunc> funcs, IStorageActionsAccessor actions, WorkContext context, IndexingWorkStats stats) { return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = () => stats.IndexingAttempts++, CancelMoveNext = () => stats.IndexingAttempts--, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); stats.IndexingErrors++; } }.RobustEnumeration(input, funcs)); }
protected void Write(WorkContext context, Func<IndexWriter, Analyzer, bool> action) { if (disposed) throw new ObjectDisposedException("Index " + name + " has been disposed"); lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List<Action>(); Analyzer analyzer = null; try { try { analyzer = CreateAnalyzer(new LowerCaseAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) indexWriter = new IndexWriter(directory, new StopAnalyzer(Version.LUCENE_29), IndexWriter.MaxFieldLength.UNLIMITED); try { shouldRecreateSearcher = action(indexWriter, analyzer); foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocumented); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } } finally { currentlyIndexDocumented.Clear(); if (analyzer != null) analyzer.Close(); foreach (Action dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) RecreateSearcher(); } }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, (indexWriter, analyzer) => { bool madeChanges = false; PropertyDescriptorCollection properties = null; var processedKeys = new HashSet<string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); var documentsWrapped = documents.Select((dynamic doc) => { if(doc.__document_id == null) throw new ArgumentException("Cannot index something which doesn't have a document id, but got: " + doc); string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) return doc; madeChanges = true; batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId), exception); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, documentId)); indexWriter.DeleteDocuments(new Term(Constants.DocumentIdFieldName, documentId.ToLowerInvariant())); return doc; }); foreach (var doc in RobustEnumerationIndex(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; IndexingResult indexingResult; if (doc is DynamicJsonObject) indexingResult = ExtractIndexDataFromDocument((DynamicJsonObject)doc); else indexingResult = ExtractIndexDataFromDocument(properties, doc); if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { var luceneDoc = new Document(); luceneDoc.Add(new Field(Constants.DocumentIdFieldName, indexingResult.NewDocId.ToLowerInvariant(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); madeChanges = true; CopyFieldsToDocument(luceneDoc, indexingResult.Fields); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.Warn( string.Format( "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId), exception); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, indexingResult.NewDocId, luceneDoc)); logIndexing.Debug("Index '{0}' resulted in: {1}", name, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } actions.Indexing.IncrementSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return madeChanges; }); logIndexing.Debug("Indexed {0} documents for {1}", count, name); }
public override void Remove(string[] keys, WorkContext context) { Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), name)); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); keys.Apply( key => batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, key), exception); context.AddError(name, key, exception.Message, "OnIndexEntryDeleted Trigger"); }, trigger => trigger.OnIndexEntryDeleted(key))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message, "Dispose Trigger"); }, batcher => batcher.Dispose()); IndexStats currentIndexStats = null; context.TransactionalStorage.Batch(accessor => currentIndexStats = accessor.Indexing.GetIndexStats(name)); return new IndexedItemsInfo { ChangedDocs = keys.Length, HighestETag = currentIndexStats.LastIndexedEtag, DeletedKeys = keys }; }); }
protected IEnumerable<object> RobustEnumerationReduce(IEnumerable<object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { // not strictly accurate, but if we get that many errors, probably an error anyway. return new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = actions.Indexing.IncrementReduceIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementReduceIndexingAttempt, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); try { actions.Indexing.IncrementReduceIndexingFailure(); } catch (Exception e) { // we don't care about error here, because it is an error on error problem logIndexing.WarnException( String.Format("Could not increment indexing failure rate for {0}", name), e); } } }.RobustEnumeration(input, func); }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { var count = 0; Write(context, (indexWriter, analyzer, stats) => { var processedKeys = new HashSet<string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); var documentsWrapped = documents.Select((dynamic doc) => { if(doc.__document_id == null) throw new ArgumentException(string.Format("Cannot index something which doesn't have a document id, but got: '{0}'", doc)); count++; string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) return doc; batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId), exception); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(documentId)); indexWriter.DeleteDocuments(new Term(Constants.DocumentIdFieldName, documentId.ToLowerInvariant())); return doc; }); var anonymousObjectToLuceneDocumentConverter = new AnonymousObjectToLuceneDocumentConverter(indexDefinition); var luceneDoc = new Document(); var documentIdField = new Field(Constants.DocumentIdFieldName, "dummy", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS); foreach (var doc in RobustEnumerationIndex(documentsWrapped, viewGenerator.MapDefinitions, actions, context, stats)) { count++; float boost; var indexingResult = GetIndexingResult(doc, anonymousObjectToLuceneDocumentConverter, out boost); if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { count += 1; luceneDoc.GetFields().Clear(); luceneDoc.SetBoost(boost); documentIdField.SetValue(indexingResult.NewDocId.ToLowerInvariant()); luceneDoc.Add(documentIdField); foreach (var field in indexingResult.Fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format( "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId), exception); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(indexingResult.NewDocId, luceneDoc)); LogIndexedDocument(indexingResult.NewDocId, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } stats.IndexingSuccesses++; } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return count; }); logIndexing.Debug("Indexed {0} documents for {1}", count, name); }
protected void Write(WorkContext context, Func <IndexWriter, Analyzer, bool> action) { if (disposed) { throw new ObjectDisposedException("Index " + name + " has been disposed"); } lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List <Action>(); Analyzer searchAnalyzer = null; try { try { searchAnalyzer = CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = CreateIndexWriter(directory); } try { shouldRecreateSearcher = action(indexWriter, searchAnalyzer); foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocuments); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } WriteTempIndexToDiskIfNeeded(context); } finally { currentlyIndexDocuments.Clear(); if (searchAnalyzer != null) { searchAnalyzer.Close(); } foreach (Action dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) { RecreateSearcher(); } } }
private bool? MoveNext(IEnumerator en, StatefulEnumerableWrapper<object> innerEnumerator, WorkContext context, DocumentStorageActions actions) { try { actions.IncrementIndexingAttempt(); var moveNext = en.MoveNext(); if (moveNext == false) actions.DecrementIndexingAttempt(); return moveNext; } catch (Exception e) { actions.IncrementIndexingFailure(); context.AddError(name, TryGetDocKey(innerEnumerator.Current), e.Message ); log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name, GetDocId(innerEnumerator)); } return null; }
protected void Write(WorkContext context, Func <IndexWriter, Analyzer, IndexingWorkStats, int> action) { if (disposed) { throw new ObjectDisposedException("Index " + name + " has been disposed"); } lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List <Action>(); Analyzer searchAnalyzer = null; try { try { searchAnalyzer = CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = CreateIndexWriter(directory); } var stats = new IndexingWorkStats(); try { var changedDocs = action(indexWriter, searchAnalyzer, stats); docCountSinceLastOptimization += changedDocs; shouldRecreateSearcher = changedDocs > 0; foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocuments); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } UpdateIndexingStats(context, stats); WriteTempIndexToDiskIfNeeded(context); if (configuration.TransactionMode == TransactionMode.Safe) { Flush(); // just make sure changes are flushed to disk } } finally { currentlyIndexDocuments.Clear(); if (searchAnalyzer != null) { searchAnalyzer.Close(); } foreach (Action dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) { RecreateSearcher(); } } }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, (indexWriter, analyzer) => { bool madeChanges = false; PropertyDescriptorCollection properties = null; var processedKeys = new HashSet <string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); var documentsWrapped = documents.Select((dynamic doc) => { if (doc.__document_id == null) { throw new ArgumentException("Cannot index something which doesn't have a document id, but got: " + doc); } string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) { return(doc); } madeChanges = true; batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, documentId)); indexWriter.DeleteDocuments(new Term(Constants.DocumentIdFieldName, documentId.ToLowerInvariant())); return(doc); }); foreach (var doc in RobustEnumerationIndex(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; IndexingResult indexingResult; if (doc is DynamicJsonObject) { indexingResult = ExtractIndexDataFromDocument((DynamicJsonObject)doc); } else { indexingResult = ExtractIndexDataFromDocument(properties, doc); } if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { var luceneDoc = new Document(); luceneDoc.Add(new Field(Constants.DocumentIdFieldName, indexingResult.NewDocId.ToLowerInvariant(), Field.Store.YES, Field.Index.NOT_ANALYZED)); madeChanges = true; CopyFieldsToDocument(luceneDoc, indexingResult.Fields); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, indexingResult.NewDocId, luceneDoc)); logIndexing.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } actions.Indexing.IncrementSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return(madeChanges); }); logIndexing.DebugFormat("Indexed {0} documents for {1}", count, name); }
public void ReduceDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> mappedResults, WorkContext context, IStorageActionsAccessor actions, string[] reduceKeys) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, (indexWriter, analyzer) => { var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); foreach (var reduceKey in reduceKeys) { var entryKey = reduceKey; indexWriter.DeleteDocuments(new Term(Raven.Abstractions.Data.Constants.ReduceKeyFieldName, entryKey.ToLowerInvariant())); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, entryKey); context.AddError(name, entryKey, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, entryKey)); } PropertyDescriptorCollection properties = null; foreach (var doc in RobustEnumerationReduce(mappedResults, viewGenerator.ReduceDefinition, actions, context)) { count++; var fields = GetFields(doc, ref properties).ToList(); string reduceKeyAsString = ExtractReduceKey(viewGenerator, doc); var luceneDoc = new Document(); luceneDoc.Add(new Field(Raven.Abstractions.Data.Constants.ReduceKeyFieldName, reduceKeyAsString.ToLowerInvariant(), Field.Store.NO, Field.Index.NOT_ANALYZED)); foreach (var field in fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, reduceKeyAsString); context.AddError(name, reduceKeyAsString, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, reduceKeyAsString, luceneDoc)); logIndexing.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKeyAsString, name, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); actions.Indexing.IncrementReduceSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return(true); }); if (logIndexing.IsDebugEnabled) { logIndexing.DebugFormat("Reduce resulted in {0} entries for {1} for reduce keys: {2}", count, name, string.Join(", ", reduceKeys)); } }
protected IEnumerable<object> RobustEnumeration(IEnumerable<object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { return new RobustEnumerator { BeforeMoveNext = actions.Indexing.IncrementIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementIndexingAttempt, OnError = (exception, o) => { actions.Indexing.IncrementIndexingFailure(); context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnFormat(exception, "Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)); } }.RobustEnumeration(input, func); }
public void ReduceDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> mappedResults, WorkContext context, IStorageActionsAccessor actions, string[] reduceKeys) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, indexWriter => { var batchers = context.IndexUpdateTriggers.Select(x=>x.CreateBatcher(name)) .Where(x=>x!=null) .ToList(); foreach (var reduceKey in reduceKeys) { var entryKey = reduceKey; indexWriter.DeleteDocuments(new Term("__reduce_key", entryKey.ToLowerInvariant())); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, entryKey); context.AddError(name, entryKey, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, entryKey)); } PropertyDescriptorCollection properties = null; foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context)) { count++; var fields = GetFields(doc, ref properties); dynamic reduceKey = viewGenerator.GroupByExtraction(doc); if (reduceKey == null) { throw new InvalidOperationException("Could not find reduce key for " + name + " in the result: " + doc); } string reduceKeyAsString = ReduceKeyToString(reduceKey); var luceneDoc = new Document(); luceneDoc.Add(new Field("__reduce_key", reduceKeyAsString.ToLowerInvariant(), Field.Store.NO, Field.Index.NOT_ANALYZED)); foreach (var field in fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, reduceKeyAsString); context.AddError(name, reduceKeyAsString, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, reduceKeyAsString, luceneDoc)); logIndexing.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKeyAsString, name, luceneDoc); indexWriter.AddDocument(luceneDoc); actions.Indexing.IncrementSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return true; }); if (logIndexing.IsDebugEnabled) { logIndexing.DebugFormat("Reduce resulted in {0} entries for {1} for reduce keys: {2}", count, name, string.Join(", ", reduceKeys)); } }
public override void Remove(string[] keys, WorkContext context) { DeletionBatchInfo deletionBatchInfo = null; try { deletionBatchInfo = context.ReportDeletionBatchStarted(PublicName, keys.Length); Write((writer, analyzer, stats) => { var indexUpdateTriggersDuration = new Stopwatch(); stats.Operation = IndexingWorkStats.Status.Ignore; if (logIndexing.IsDebugEnabled) { logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), PublicName)); } List <AbstractIndexUpdateTriggerBatcher> batchers; using (StopwatchScope.For(indexUpdateTriggersDuration)) { batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); keys.Apply(key => InvokeOnIndexEntryDeletedOnAllBatchers(batchers, new Term(Constants.DocumentIdFieldName, key.ToLowerInvariant()))); } var deleteDocumentsDuration = new Stopwatch(); using (StopwatchScope.For(deleteDocumentsDuration)) { writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); } deletionBatchInfo.PerformanceStats.Add(PerformanceStats.From(IndexingOperation.Delete_Documents, deleteDocumentsDuration.ElapsedMilliseconds)); using (StopwatchScope.For(indexUpdateTriggersDuration)) { batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger in " + PublicName, e); context.AddError(indexId, PublicName, null, e, "Dispose Trigger"); }, batcher => batcher.Dispose()); } deletionBatchInfo.PerformanceStats.Add(PerformanceStats.From(IndexingOperation.Delete_IndexUpdateTriggers, indexUpdateTriggersDuration.ElapsedMilliseconds)); return(new IndexedItemsInfo(GetLastEtagFromStats()) { ChangedDocs = keys.Length, DeletedKeys = keys }); }, deletionBatchInfo.PerformanceStats); } finally { if (deletionBatchInfo != null) { context.ReportDeletionBatchCompleted(deletionBatchInfo); } } }
protected IEnumerable<object> RobustEnumerationIndex(IEnumerable<object> input, IEnumerable<IndexingFunc> funcs, IStorageActionsAccessor actions, WorkContext context, IndexingWorkStats stats) { return new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = () => stats.IndexingAttempts++, CancelMoveNext = () => stats.IndexingAttempts--, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); stats.IndexingErrors++; } }.RobustEnumeration(input, funcs); }
protected void Write(WorkContext context, Func<IndexWriter, bool> action) { if (disposed) throw new ObjectDisposedException("Index " + name + " has been disposed"); lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List<Action>(); Analyzer analyzer = null; try { analyzer = CreateAnalyzer(toDispose); var indexWriter = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED); try { shouldRecreateSearcher = action(indexWriter); } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } finally { indexWriter.Close(); } } finally { if (analyzer != null) analyzer.Close(); foreach (var dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) RecreateSearcher(); } }
// This method may be called concurrently, by both the ReduceTask (for removal) // and by the ReducingExecuter (for add/modify). This is okay with us, since the // Write() call is already handling locking properly public void ReduceDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> mappedResults, WorkContext context, IStorageActionsAccessor actions, string[] reduceKeys) { var count = 0; Write(context, (indexWriter, analyzer) => { actions.Indexing.SetCurrentIndexStatsTo(name); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); foreach (var reduceKey in reduceKeys) { var entryKey = reduceKey; indexWriter.DeleteDocuments(new Term(Abstractions.Data.Constants.ReduceKeyFieldName, entryKey.ToLowerInvariant())); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, entryKey), exception); context.AddError(name, entryKey, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, entryKey)); } PropertyDescriptorCollection properties = null; foreach (var doc in RobustEnumerationReduce(mappedResults, viewGenerator.ReduceDefinition, actions, context)) { count++; var fields = GetFields(doc, ref properties).ToList(); string reduceKeyAsString = ExtractReduceKey(viewGenerator, doc); var luceneDoc = new Document(); luceneDoc.Add(new Field(Abstractions.Data.Constants.ReduceKeyFieldName, reduceKeyAsString.ToLowerInvariant(), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS)); foreach (var field in fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, reduceKeyAsString), exception); context.AddError(name, reduceKeyAsString, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, reduceKeyAsString, luceneDoc)); logIndexing.Debug("Reduce key {0} result in index {1} gave document: {2}", reduceKeyAsString, name, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); actions.Indexing.IncrementReduceSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return true; }); logIndexing.Debug(() => string.Format("Reduce resulted in {0} entries for {1} for reduce keys: {2}", count, name, string.Join(", ", reduceKeys))); }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IndexingBatch batch, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { var count = 0; var sourceCount = 0; var sw = Stopwatch.StartNew(); var start = SystemTime.UtcNow; Write((indexWriter, analyzer, stats) => { var processedKeys = new HashSet<string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); try { var docIdTerm = new Term(Constants.DocumentIdFieldName); var documentsWrapped = batch.Docs.Select((doc,i) => { Interlocked.Increment(ref sourceCount); if (doc.__document_id == null) throw new ArgumentException( string.Format("Cannot index something which doesn't have a document id, but got: '{0}'", doc)); string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) return doc; batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId), exception); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(documentId)); if (batch.SkipDeleteFromIndex[i] == false || context.ShouldRemoveFromIndex(documentId)) // maybe it is recently deleted? indexWriter.DeleteDocuments(docIdTerm.CreateTerm(documentId.ToLowerInvariant())); return doc; }) .Where(x => x is FilteredDocument == false) .ToList(); BackgroundTaskExecuter.Instance.ExecuteAllBuffered(context, documentsWrapped, (partition) => { var anonymousObjectToLuceneDocumentConverter = new AnonymousObjectToLuceneDocumentConverter(indexDefinition); var luceneDoc = new Document(); var documentIdField = new Field(Constants.DocumentIdFieldName, "dummy", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); foreach (var doc in RobustEnumerationIndex(partition, viewGenerator.MapDefinitions, actions, stats)) { float boost; var indexingResult = GetIndexingResult(doc, anonymousObjectToLuceneDocumentConverter, out boost); if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { Interlocked.Increment(ref count); luceneDoc.GetFields().Clear(); luceneDoc.Boost = boost; documentIdField.SetValue(indexingResult.NewDocId.ToLowerInvariant()); luceneDoc.Add(documentIdField); foreach (var field in indexingResult.Fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId), exception); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(indexingResult.NewDocId, luceneDoc)); LogIndexedDocument(indexingResult.NewDocId, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } Interlocked.Increment(ref stats.IndexingSuccesses); } }); } catch(Exception e) { batchers.ApplyAndIgnoreAllErrors( ex => { logIndexing.WarnException("Failed to notify index update trigger batcher about an error", ex); context.AddError(name, null, ex.Message); }, x => x.AnErrorOccured(e)); throw; } finally { batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); } return sourceCount; }); AddindexingPerformanceStat(new IndexingPerformanceStats { OutputCount = count, InputCount = sourceCount, Duration = sw.Elapsed, Operation = "Index", Started = start }); logIndexing.Debug("Indexed {0} documents for {1}", count, name); }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IndexingBatch batch, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { var count = 0; var sourceCount = 0; var sw = Stopwatch.StartNew(); var start = SystemTime.UtcNow; Write(context, (indexWriter, analyzer, stats) => { var processedKeys = new HashSet <string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); try { var docIdTerm = new Term(Constants.DocumentIdFieldName); var documentsWrapped = batch.Docs.Select((doc, i) => { Interlocked.Increment(ref sourceCount); if (doc.__document_id == null) { throw new ArgumentException( string.Format("Cannot index something which doesn't have a document id, but got: '{0}'", doc)); } string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) { return(doc); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId), exception); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(documentId)); if (batch.SkipDeleteFromIndex[i] == false) { indexWriter.DeleteDocuments(docIdTerm.CreateTerm(documentId.ToLowerInvariant())); } return(doc); }) .Where(x => x is FilteredDocument == false) .ToList(); BackgroundTaskExecuter.Instance.ExecuteAllBuffered(context, documentsWrapped, (partition) => { var anonymousObjectToLuceneDocumentConverter = new AnonymousObjectToLuceneDocumentConverter(indexDefinition); var luceneDoc = new Document(); var documentIdField = new Field(Constants.DocumentIdFieldName, "dummy", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); foreach (var doc in RobustEnumerationIndex(partition, viewGenerator.MapDefinitions, actions, stats)) { float boost; var indexingResult = GetIndexingResult(doc, anonymousObjectToLuceneDocumentConverter, out boost); if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { Interlocked.Increment(ref count); luceneDoc.GetFields().Clear(); luceneDoc.Boost = boost; documentIdField.SetValue(indexingResult.NewDocId.ToLowerInvariant()); luceneDoc.Add(documentIdField); foreach (var field in indexingResult.Fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId), exception); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(indexingResult.NewDocId, luceneDoc)); LogIndexedDocument(indexingResult.NewDocId, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } Interlocked.Increment(ref stats.IndexingSuccesses); } }); } catch (Exception e) { batchers.ApplyAndIgnoreAllErrors( ex => { logIndexing.WarnException("Failed to notify index update trigger batcher about an error", ex); context.AddError(name, null, ex.Message); }, x => x.AnErrorOccured(e)); throw; } finally { batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); } return(sourceCount); }); AddindexingPerformanceStat(new IndexingPerformanceStats { OutputCount = count, InputCount = sourceCount, Duration = sw.Elapsed, Operation = "Index", Started = start }); logIndexing.Debug("Indexed {0} documents for {1}", count, name); }
// This method may be called concurrently, by both the ReduceTask (for removal) // and by the ReducingExecuter (for add/modify). This is okay with us, since the // Write() call is already handling locking properly public void ReduceDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> mappedResults, WorkContext context, IStorageActionsAccessor actions, string[] reduceKeys) { var count = 0; Write(context, (indexWriter, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Reduce; var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); foreach (var reduceKey in reduceKeys) { var entryKey = reduceKey; indexWriter.DeleteDocuments(new Term(Abstractions.Data.Constants.ReduceKeyFieldName, entryKey.ToLowerInvariant())); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, entryKey), exception); context.AddError(name, entryKey, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(entryKey)); } PropertyDescriptorCollection properties = null; var anonymousObjectToLuceneDocumentConverter = new AnonymousObjectToLuceneDocumentConverter(indexDefinition); var luceneDoc = new Document(); var reduceKeyField = new Field(Constants.ReduceKeyFieldName, "dummy", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS); foreach (var doc in RobustEnumerationReduce(mappedResults, viewGenerator.ReduceDefinition, actions, context, stats)) { count++; float boost; var fields = GetFields(anonymousObjectToLuceneDocumentConverter, doc, ref properties, out boost).ToList(); string reduceKeyAsString = ExtractReduceKey(viewGenerator, doc); reduceKeyField.SetValue(reduceKeyAsString.ToLowerInvariant()); luceneDoc.GetFields().Clear(); luceneDoc.SetBoost(boost); luceneDoc.Add(reduceKeyField); foreach (var field in fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, reduceKeyAsString), exception); context.AddError(name, reduceKeyAsString, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(reduceKeyAsString, luceneDoc)); LogIndexedDocument(reduceKeyAsString, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); stats.ReduceSuccesses++; } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return(count + reduceKeys.Length); }); logIndexing.Debug(() => string.Format("Reduce resulted in {0} entries for {1} for reduce keys: {2}", count, name, string.Join(", ", reduceKeys))); }
public override void Remove(string[] keys, WorkContext context) { Write(context, (writer, analyzer,stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), name)); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); keys.Apply( key => batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, key), exception); context.AddError(name, key, exception.Message); }, trigger => trigger.OnIndexEntryDeleted(key))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, batcher => batcher.Dispose()); return keys.Length; }); }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { var count = 0; Write(context, (indexWriter, analyzer, stats) => { var processedKeys = new HashSet <string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); var documentsWrapped = documents.Select((dynamic doc) => { if (doc.__document_id == null) { throw new ArgumentException(string.Format("Cannot index something which doesn't have a document id, but got: '{0}'", doc)); } count++; string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) { return(doc); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId), exception); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(documentId)); indexWriter.DeleteDocuments(new Term(Constants.DocumentIdFieldName, documentId.ToLowerInvariant())); return(doc); }); var anonymousObjectToLuceneDocumentConverter = new AnonymousObjectToLuceneDocumentConverter(indexDefinition); var luceneDoc = new Document(); var documentIdField = new Field(Constants.DocumentIdFieldName, "dummy", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS); foreach (var doc in RobustEnumerationIndex(documentsWrapped, viewGenerator.MapDefinitions, actions, context, stats)) { count++; float boost; var indexingResult = GetIndexingResult(doc, anonymousObjectToLuceneDocumentConverter, out boost); if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { count += 1; luceneDoc.GetFields().Clear(); luceneDoc.SetBoost(boost); documentIdField.SetValue(indexingResult.NewDocId.ToLowerInvariant()); luceneDoc.Add(documentIdField); foreach (var field in indexingResult.Fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnException( string.Format("Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId), exception); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(indexingResult.NewDocId, luceneDoc)); LogIndexedDocument(indexingResult.NewDocId, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } stats.IndexingSuccesses++; } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return(count); }); logIndexing.Debug("Indexed {0} documents for {1}", count, name); }
protected void Write(WorkContext context, Func<IndexWriter, Analyzer, bool> action) { if (disposed) throw new ObjectDisposedException("Index " + name + " has been disposed"); lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List<Action>(); Analyzer searchAnalyzer = null; try { try { searchAnalyzer = CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = CreateIndexWriter(directory); } try { shouldRecreateSearcher = action(indexWriter, searchAnalyzer); foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocuments); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } WriteTempIndexToDiskIfNeeded(context); } finally { currentlyIndexDocuments.Clear(); if (searchAnalyzer != null) searchAnalyzer.Close(); foreach (Action dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) RecreateSearcher(); } }
public override void Remove(string[] keys, WorkContext context) { Write(context, writer => { if (logIndexing.IsDebugEnabled) { logIndexing.DebugFormat("Deleting ({0}) from {1}", string.Format(", ", keys), name); } var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); keys.Apply( key => batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, key); context.AddError(name, key, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, key))); writer.DeleteDocuments(keys.Select(k => new Term("__document_id", k)).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message ); }, batcher => batcher.Dispose()); return true; }); }
// we don't care about tracking map/reduce stats here, since it is merely // an optimization step protected IEnumerable<object> RobustEnumerationReduceDuringMapPhase(IEnumerable<object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { // not strictly accurate, but if we get that many errors, probably an error anyway. return new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = () => { }, // don't care CancelMoveNext = () => { }, // don't care OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); } }.RobustEnumeration(input, func); }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, indexWriter => { bool madeChanges = false; PropertyDescriptorCollection properties = null; var processedKeys = new HashSet<string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); var documentsWrapped = documents.Select((dynamic doc) => { string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) return doc; madeChanges = true; batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, documentId)); indexWriter.DeleteDocuments(new Term("__document_id", documentId.ToLowerInvariant())); return doc; }); foreach (var doc in RobustEnumeration(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; string newDocId; IEnumerable<AbstractField> fields; if (doc is DynamicJsonObject) fields = ExtractIndexDataFromDocument((DynamicJsonObject)doc, out newDocId); else fields = ExtractIndexDataFromDocument(properties, doc, out newDocId); if (newDocId != null) { var luceneDoc = new Document(); luceneDoc.Add(new Field("__document_id", newDocId.ToLowerInvariant(), Field.Store.YES, Field.Index.NOT_ANALYZED)); madeChanges = true; CopyFieldsToDocument(luceneDoc, fields); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, newDocId); context.AddError(name, newDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, newDocId, luceneDoc)); logIndexing.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc); indexWriter.AddDocument(luceneDoc); } actions.Indexing.IncrementSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return madeChanges; }); logIndexing.DebugFormat("Indexed {0} documents for {1}", count, name); }
public override void Remove(string[] keys, WorkContext context) { Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), indexId)); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); keys.Apply( key => InvokeOnIndexEntryDeletedOnAllBatchers(batchers, new Term(Constants.DocumentIdFieldName, key))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.DocumentIdFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger", e); context.AddError(indexId, null, e.Message, "Dispose Trigger"); }, batcher => batcher.Dispose()); return new IndexedItemsInfo(GetLastEtagFromStats()) { ChangedDocs = keys.Length, DeletedKeys = keys }; }); }
protected void Write(WorkContext context, Func<IndexWriter, Analyzer, IndexingWorkStats, int> action) { if (disposed) throw new ObjectDisposedException("Index " + name + " has been disposed"); LastIndexTime = SystemTime.UtcNow; lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List<Action>(); Analyzer searchAnalyzer = null; try { waitReason = "Write"; try { searchAnalyzer = CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = CreateIndexWriter(directory); } var locker = directory.MakeLock("writing-to-index.lock"); try { var stats = new IndexingWorkStats(); try { var changedDocs = action(indexWriter, searchAnalyzer, stats); docCountSinceLastOptimization += changedDocs; shouldRecreateSearcher = changedDocs > 0; foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocuments); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } UpdateIndexingStats(context, stats); WriteTempIndexToDiskIfNeeded(context); Flush(); // just make sure changes are flushed to disk } finally { locker.Release(); } } finally { currentlyIndexDocuments.Clear(); if (searchAnalyzer != null) searchAnalyzer.Close(); foreach (Action dispose in toDispose) { dispose(); } waitReason = null; LastIndexTime = SystemTime.UtcNow; } if (shouldRecreateSearcher) RecreateSearcher(); } }
public override void Remove(string[] keys, WorkContext context) { context.TransactionalStorage.Batch(actions => { var reduceKeyAndBuckets = new Dictionary<ReduceKeyAndBucket, int>(); foreach (var key in keys) { actions.MapReduce.DeleteMappedResultsForDocumentId(key, indexId, reduceKeyAndBuckets); } actions.MapReduce.UpdateRemovedMapReduceStats(indexId, reduceKeyAndBuckets); foreach (var reduceKeyAndBucket in reduceKeyAndBuckets) { actions.MapReduce.ScheduleReductions(indexId, 0, reduceKeyAndBucket.Key); } }); Write((writer, analyzer, stats) => { stats.Operation = IndexingWorkStats.Status.Ignore; if (logIndexing.IsDebugEnabled) logIndexing.Debug(() => string.Format("Deleting ({0}) from {1}", string.Join(", ", keys), PublicName)); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); keys.Apply( key => InvokeOnIndexEntryDeletedOnAllBatchers(batchers, new Term(Constants.ReduceKeyFieldName, key.ToLowerInvariant()))); writer.DeleteDocuments(keys.Select(k => new Term(Constants.ReduceKeyFieldName, k.ToLowerInvariant())).ToArray()); batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.WarnException("Failed to dispose on index update trigger in " + PublicName, e); context.AddError(indexId, PublicName, null, e, "Dispose Trigger"); }, batcher => batcher.Dispose()); return new IndexedItemsInfo(null) { ChangedDocs = keys.Length }; }); }
protected IEnumerable<object> RobustEnumerationReduce(IEnumerable<object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { return new RobustEnumerator { BeforeMoveNext = actions.Indexing.IncrementReduceIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementReduceIndexingAttempt, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnFormat(exception, "Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)); try { actions.Indexing.IncrementReduceIndexingFailure(); } catch (Exception e) { // we don't care about error here, because it is an error on error problem logIndexing.WarnFormat(e, "Could not increment indexing failure rate for {0}", name); } } }.RobustEnumeration(input, func); }