public SuggestionQueryIndexExtension( WorkContext workContext, string key, StringDistance distanceType, bool isRunInMemory, string field, float accuracy) { this.workContext = workContext; this.key = key; this.field = field; if (isRunInMemory) { directory = new RAMDirectory(); } else { directory = FSDirectory.Open(new DirectoryInfo(key)); } this.spellChecker = new SpellChecker.Net.Search.Spell.SpellChecker(directory, null); this.spellChecker.SetAccuracy(accuracy); this.spellChecker.setStringDistance(distanceType); }
public PrefetchingBehavior(PrefetchingUser prefetchingUser, WorkContext context, BaseBatchSizeAutoTuner autoTuner) { this.context = context; this.autoTuner = autoTuner; PrefetchingUser = prefetchingUser; MemoryStatistics.RegisterLowMemoryHandler(this); }
public override void Execute(WorkContext context) { if (logger.IsDebugEnabled) { logger.Debug("Going to touch the following documents (missing references, need to check for concurrent transactions): {0}", string.Join(", ", Keys)); } context.TransactionalStorage.Batch(accessor => { foreach (var key in Keys) { foreach (var index in context.IndexStorage.Indexes) { var set = context.DoNotTouchAgainIfMissingReferences.GetOrAdd(index, _ => new ConcurrentSet<string>(StringComparer.OrdinalIgnoreCase)); set.Add(key); } try { Etag preTouchEtag; Etag afterTouchEtag; accessor.Documents.TouchDocument(key, out preTouchEtag, out afterTouchEtag); } catch (ConcurrencyException) { } } }); }
public override void Execute(WorkContext context) { foreach (var indexName in context.IndexDefinitionStorage.IndexNames) { context.IndexStorage.RemoveFromIndex(indexName, Keys, context); } }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable<dynamic> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Func<object, object> documentIdFetcher = null; var reduceKeys = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); var documentsWrapped = documents.Select(doc => { var documentId = doc.__document_id; foreach (var reduceKey in actions.MappedResults.DeleteMappedResultsForDocumentId((string)documentId, name)) { reduceKeys.Add(reduceKey); } return doc; }); foreach (var doc in RobustEnumeration(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc); var docIdValue = documentIdFetcher(doc); if (docIdValue == null) throw new InvalidOperationException("Could not find document id for this document"); var reduceValue = viewGenerator.GroupByExtraction(doc); if (reduceValue == null) { logIndexing.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue); continue; } var reduceKey = ReduceKeyToString(reduceValue); var docId = docIdValue.ToString(); reduceKeys.Add(reduceKey); var data = GetMapedData(doc); logIndexing.DebugFormat("Mapped result for '{0}': '{1}'", name, data); var hash = ComputeHash(name, reduceKey); actions.MappedResults.PutMappedResult(name, docId, reduceKey, data, hash); actions.Indexing.IncrementSuccessIndexing(); } actions.Tasks.AddTask(new ReduceTask { Index = name, ReduceKeys = reduceKeys.ToArray() }, minimumTimestamp); logIndexing.DebugFormat("Mapped {0} documents for {1}", count, name); }
public override void Execute(WorkContext context) { if (ReduceKeys.Length == 0) return; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(Index); if (viewGenerator == null) return; // deleted view? context.TransactionaStorage.Batch(actions => { log.Debug("Starting to read {0} reduce keys for index {1}", ReduceKeys.Length, Index); var itemsToFind = ReduceKeys .Select(reduceKey => new GetMappedResultsParams(Index, reduceKey, MapReduceIndex.ComputeHash(Index, reduceKey))) .ToArray(); var mappedResults = actions.MappedResults.GetMappedResults(itemsToFind) .Select(JsonToExpando.Convert); var sp = Stopwatch.StartNew(); var results = mappedResults.ToArray(); log.Debug("Read {0} reduce keys in {1} with {2} results for index {3}", ReduceKeys.Length, sp.Elapsed, results.Length, Index); sp = Stopwatch.StartNew(); context.IndexStorage.Reduce(Index, viewGenerator, results, context, actions, ReduceKeys); log.Debug("Indexed {0} reduce keys in {1} with {2} results for index {3}", ReduceKeys.Length, sp.Elapsed, results.Length, Index); }); }
public PrefetchingBehavior(PrefetchingUser prefetchingUser, WorkContext context, BaseBatchSizeAutoTuner autoTuner, string prefetchingUserDescription, bool isDefault = false, Func<int> getPrefetchintBehavioursCount = null, Func<PrefetchingSummary> getPrefetcherSummary = null) { this.context = context; this.autoTuner = autoTuner; PrefetchingUser = prefetchingUser; this.userDescription = prefetchingUserDescription; this.IsDefault = isDefault; this.getPrefetchintBehavioursCount = getPrefetchintBehavioursCount ?? (() => 1); this.getPrefetcherSummary = getPrefetcherSummary ?? GetSummary; MemoryStatistics.RegisterLowMemoryHandler(this); LastTimeUsed = DateTime.MinValue; ingestMeter = context.MetricsCounters.DbMetrics.Meter("metrics", "ingest/sec", "In memory documents held by this prefetcher", TimeUnit.Seconds); returnedDocsMeter = context.MetricsCounters.DbMetrics.Meter("metrics", "returned docs/sec", "Documents being served by this prefetcher", TimeUnit.Seconds); if (isDefault) { context.Database.TransactionalStorage.Batch(accessor => { recentEtag = accessor.Staleness.GetMostRecentDocumentEtag(); }); } }
protected AbstractIndexingExecuter( ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler) { this.transactionalStorage = transactionalStorage; this.context = context; this.scheduler = scheduler; }
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, id, indexDefinition, viewGenerator, context) { jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer(); jsonSerializer.Converters = MapReduceConverters; }
public override void Execute(WorkContext context) { if (ReduceKeys.Length == 0) return; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(Index); if (viewGenerator == null) return; // deleted view? context.TransactionaStorage.Batch(actions => { IEnumerable<object> mappedResults = null; foreach (var reduceKey in ReduceKeys) { IEnumerable<object> enumerable = actions.MappedResults.GetMappedResults(Index, reduceKey, MapReduceIndex.ComputeHash(Index, reduceKey)) .Select(JsonToExpando.Convert); if (mappedResults == null) mappedResults = enumerable; else mappedResults = mappedResults.Concat(enumerable); } context.IndexStorage.Reduce(Index, viewGenerator, mappedResults, context, actions, ReduceKeys); }); }
protected AbstractIndexingExecuter(WorkContext context, IndexReplacer indexReplacer) { this.transactionalStorage = context.TransactionalStorage; this.context = context; this.indexReplacer = indexReplacer; this.scheduler = context.TaskScheduler; }
public IndexingExecuter(WorkContext context, DatabaseEtagSynchronizer synchronizer, Prefetcher prefetcher) : base(context) { autoTuner = new IndexBatchSizeAutoTuner(context); etagSynchronizer = synchronizer.GetSynchronizer(EtagSynchronizerType.Indexer); prefetchingBehavior = prefetcher.GetPrefetchingBehavior(PrefetchingUser.Indexer, autoTuner); }
protected BaseBatchSizeAutoTuner(WorkContext context) { this.context = context; NumberOfItemsToIndexInSingleBatch = InitialNumberOfItems; currentlyUsedBatchSizes = new ConcurrentDictionary<Guid, long>(); memoryLimitForIndexingInBytes = context.Configuration.MemoryLimitForIndexingInMB * 1024 * 1024; }
public IndexingExecuter(WorkContext context, Prefetcher prefetcher) : base(context) { autoTuner = new IndexBatchSizeAutoTuner(context); this.prefetcher = prefetcher; defaultPrefetchingBehavior = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner); prefetchingBehaviors.TryAdd(defaultPrefetchingBehavior); }
protected AbstractIndexingExecuter(WorkContext context, IndexReplacer indexReplacer) { Log = LogManager.GetLogger(GetType()); this.transactionalStorage = context.TransactionalStorage; this.context = context; this.indexReplacer = indexReplacer; this.scheduler = context.TaskScheduler; }
public override void Execute(WorkContext context) { if (logger.IsDebugEnabled) { logger.Debug("Going to touch the following documents (LoadDocument references, need to check for concurrent transactions): {0}", string.Join(", ", ReferencesToCheck)); } using (context.Database.TransactionalStorage.DisableBatchNesting()) { var docsToTouch = new HashSet<string>(StringComparer.OrdinalIgnoreCase); context.TransactionalStorage.Batch(accessor => { foreach (var kvp in ReferencesToCheck) { var doc = accessor.Documents.DocumentMetadataByKey(kvp.Key, null); if (doc == null) { logger.Debug("Cannot touch {0}, non existant document", kvp.Key); continue; } if (doc.Etag == kvp.Value) { logger.Debug("Don't need to touch {0}, etag {1} is the same as when we last saw it", kvp.Key, doc.Etag); continue; } docsToTouch.Add(kvp.Key); } }); using (context.Database.DocumentLock.Lock()) { context.TransactionalStorage.Batch(accessor => { foreach (var doc in docsToTouch) { try { Etag preTouchEtag; Etag afterTouchEtag; accessor.Documents.TouchDocument(doc, out preTouchEtag, out afterTouchEtag); logger.Debug("Touching document: {0}, etag before touch: {1}, after touch {2}", doc, preTouchEtag, afterTouchEtag); } catch (ConcurrencyException) { logger.Info("Concurrency exception when touching {0}", doc); } context.Database.CheckReferenceBecauseOfDocumentUpdate(doc, accessor); } }); } } }
public IndexingExecuter(WorkContext context, Prefetcher prefetcher, IndexReplacer indexReplacer) : base(context, indexReplacer) { autoTuner = new IndexBatchSizeAutoTuner(context); this.prefetcher = prefetcher; defaultPrefetchingBehavior = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner, "Default Prefetching behavior", true); defaultPrefetchingBehavior.ShouldHandleUnusedDocumentsAddedAfterCommit = true; prefetchingBehaviors.TryAdd(defaultPrefetchingBehavior); }
public DocumentDatabase(InMemoryRavenConfiguration configuration) { Configuration = configuration; configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers }; dynamicQueryRunner = new DynamicQueryRunner(this); suggestionQueryRunner = new SuggestionQueryRunner(this); TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications); configuration.Container.SatisfyImportsOnce(TransactionalStorage); bool newDb; try { newDb = TransactionalStorage.Initialize(this); } catch (Exception) { TransactionalStorage.Dispose(); throw; } TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag")); IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration); workContext.IndexStorage = IndexStorage; workContext.TransactionaStorage = TransactionalStorage; workContext.IndexDefinitionStorage = IndexDefinitionStorage; try { InitializeTriggers(); ExecuteStartupTasks(); } catch (Exception) { Dispose(); throw; } if (!newDb) return; OnNewlyCreatedDatabase(); }
protected BaseBatchSizeAutoTuner(WorkContext context) { this.context = context; NumberOfItemsToIndexInSingleBatch = InitialNumberOfItems; currentlyUsedBatchSizes = new ConcurrentDictionary<Guid, long>(); memoryLimitForIndexingInBytes = context.Configuration.MemoryLimitForIndexingInMB * 1024 * 1024; FetchingDocumentsFromDiskTimeout = TimeSpan.FromSeconds(context.Configuration.FetchingDocumentsFromDiskTimeoutInSeconds); maximumSizeAllowedToFetchFromStorageInMb = context.Configuration.MaximumSizeAllowedToFetchFromStorageInMb; }
public DocumentDatabase(InMemroyRavenConfiguration configuration) { Configuration = configuration; configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers }; dynamicQueryRunner = new DynamicQueryRunner(this); TransactionalStorage = configuration.CreateTransactionalStorage(workContext.NotifyAboutWork); configuration.Container.SatisfyImportsOnce(TransactionalStorage); bool newDb; try { newDb = TransactionalStorage.Initialize(); } catch (Exception) { TransactionalStorage.Dispose(); throw; } IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration); workContext.PerformanceCounters = new PerformanceCounters("Instance @ " + configuration.Port); workContext.IndexStorage = IndexStorage; workContext.TransactionaStorage = TransactionalStorage; workContext.IndexDefinitionStorage = IndexDefinitionStorage; try { InitializeTriggers(); ExecuteStartupTasks(); } catch (Exception) { Dispose(); throw; } if (!newDb) return; OnNewlyCreatedDatabase(); }
public override void Execute(WorkContext context) { var keysToRemove = new HashSet<string>(); context.TransactionaStorage.Batch(accessor => { keysToRemove = new HashSet<string>(Keys.Where(key=>accessor.Documents.DocumentMetadataByKey(key, null) == null)); accessor.Indexing.TouchIndexEtag(Index); }); context.IndexStorage.RemoveFromIndex(Index, keysToRemove.ToArray(), context); }
public MapReduceIndex(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, name, indexDefinition, viewGenerator, context) { jsonSerializer = new JsonSerializer(); foreach (var jsonConverter in Default.Converters) { jsonSerializer.Converters.Add(jsonConverter); } }
public override void Execute(WorkContext context) { var keysToRemove = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); context.TransactionaStorage.Batch(accessor => { keysToRemove = new HashSet<string>(Keys.Where(key=>FilterDocuments(context, accessor, key))); accessor.Indexing.TouchIndexEtag(Index); }); context.IndexStorage.RemoveFromIndex(Index, keysToRemove.ToArray(), context); }
protected BaseBatchSizeAutoTuner(WorkContext context) { this.context = context; FetchingDocumentsFromDiskTimeout = TimeSpan.FromSeconds(context.Configuration.Prefetcher.FetchingDocumentsFromDiskTimeoutInSeconds); maximumSizeAllowedToFetchFromStorageInMb = context.Configuration.Prefetcher.MaximumSizeAllowedToFetchFromStorageInMb; // ReSharper disable once DoNotCallOverridableMethodsInConstructor NumberOfItemsToProcessInSingleBatch = InitialNumberOfItems; MemoryStatistics.RegisterLowMemoryHandler(this); _currentlyUsedBatchSizesInBytes = new ConcurrentDictionary<Guid, long>(); }
public override void Execute(WorkContext context) { if (logger.IsDebugEnabled) { logger.Debug("Going to touch the following documents (missing references, need to check for concurrent transactions): {0}", string.Join(", ", MissingReferences)); } context.TransactionalStorage.Batch(accessor => { foreach (var docWithMissingRef in MissingReferences) { foreach (var index in context.IndexStorage.Indexes) { var set = context.DoNotTouchAgainIfMissingReferences.GetOrAdd(index, _ => new ConcurrentSet<string>(StringComparer.OrdinalIgnoreCase)); set.Add(docWithMissingRef.Key); } bool foundReference = false; using (context.TransactionalStorage.DisableBatchNesting()) { context.TransactionalStorage.Batch(freshAccessor => { foreach (var missingRef in docWithMissingRef.Value) { var doc = freshAccessor.Documents.DocumentMetadataByKey(missingRef, null); if (doc == null) continue; foundReference = true; break; } }); } if(foundReference == false) continue; try { using (context.Database.DocumentLock.Lock()) { Etag preTouchEtag; Etag afterTouchEtag; accessor.Documents.TouchDocument(docWithMissingRef.Key, out preTouchEtag, out afterTouchEtag); } } catch (ConcurrencyException) { } } }); }
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, id, indexDefinition, viewGenerator, context) { jsonSerializer = new JsonSerializer(); foreach (var jsonConverter in Default.Converters) { jsonSerializer.Converters.Add(jsonConverter); } jsonSerializer.Converters.Add(new IgnoreFieldable()); }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, DocumentStorageActions actions) { actions.SetCurrentIndexStatsTo(name); var count = 0; PropertyDescriptor groupByPropertyDescriptor = null; PropertyDescriptor documentIdPropertyDescriptor = null; var reduceKeys = new HashSet<string>(); foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context)) { count++; if (groupByPropertyDescriptor == null) { var props = TypeDescriptor.GetProperties(doc); groupByPropertyDescriptor = props.Find(viewGenerator.GroupByField, false); documentIdPropertyDescriptor = props.Find("__document_id", false); } var docIdValue = documentIdPropertyDescriptor.GetValue(doc); if (docIdValue == null) throw new InvalidOperationException("Could not find document id for this document"); var reduceValue = groupByPropertyDescriptor.GetValue(doc); if (reduceValue == null) { log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByField, docIdValue); continue; } var reduceKey = reduceValue.ToString(); var docId = docIdValue.ToString(); reduceKeys.Add(reduceKey); actions.PutMappedResult(name, docId, reduceKey, JObject.FromObject(doc).ToString(Formatting.None)); actions.IncrementSuccessIndexing(); } foreach (var reduceKey in reduceKeys) { actions.AddTask(new ReduceTask { Index = name, ReduceKey = reduceKey }); } log.DebugFormat("Mapped {0} documents for {1}", count, name); }
public override void Remove(string[] keys, WorkContext context) { Write(writer => { if (log.IsDebugEnabled) { log.DebugFormat("Deleting ({0}) from {1}", string.Format(", ", keys), name); } writer.DeleteDocuments(keys.Select(k => new Term("__document_id", k)).ToArray()); return true; }); }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, IStorageActionsAccessor actions) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(indexWriter => { bool madeChanges = false; PropertyDescriptorCollection properties = null; var processedKeys = new HashSet<string>(); var documentsWrapped = documents.Select((dynamic doc) => { var documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) return doc; madeChanges = true; context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, documentId)); indexWriter.DeleteDocuments(new Term("__document_id", documentId)); return doc; }); foreach (var doc in RobustEnumeration(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; string newDocId; IEnumerable<AbstractField> fields; if (doc is DynamicJsonObject) fields = ExtractIndexDataFromDocument((DynamicJsonObject) doc, out newDocId); else fields = ExtractIndexDataFromDocument(properties, doc, out newDocId); if (newDocId != null) { var luceneDoc = new Document(); luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.NOT_ANALYZED)); madeChanges = true; CopyFieldsToDocument(luceneDoc, fields); context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, newDocId, luceneDoc)); log.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc); indexWriter.AddDocument(luceneDoc); } actions.Indexing.IncrementSuccessIndexing(); } return madeChanges; }); log.DebugFormat("Indexed {0} documents for {1}", count, name); }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, DocumentStorageActions actions) { actions.SetCurrentIndexStatsTo(name); var count = 0; Func<object, object> documentIdFetcher = null; var reduceKeys = new HashSet<string>(); foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context)) { count++; documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc); var docIdValue = documentIdFetcher(doc); if (docIdValue == null) throw new InvalidOperationException("Could not find document id for this document"); var reduceValue = viewGenerator.GroupByExtraction(doc); if (reduceValue == null) { log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue); continue; } var reduceKey = ReduceKeyToString(reduceValue); var docId = docIdValue.ToString(); reduceKeys.Add(reduceKey); string data = GetMapedData(doc); log.DebugFormat("Mapped result for '{0}': '{1}'", name, data); var hash = ComputeHash(name, reduceKey); actions.PutMappedResult(name, docId, reduceKey, data, hash); actions.IncrementSuccessIndexing(); } foreach (var reduceKey in reduceKeys) { actions.AddTask(new ReduceTask { Index = name, ReduceKey = reduceKey }); } log.DebugFormat("Mapped {0} documents for {1}", count, name); }
public SimpleIndex(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, name, indexDefinition, viewGenerator, context) { }
public void Index(string index, AbstractViewGenerator viewGenerator, IEnumerable <dynamic> docs, WorkContext context, IStorageActionsAccessor actions) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index); return; } value.IndexDocuments(viewGenerator, docs, context, actions); }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable <dynamic> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Func <object, object> documentIdFetcher = null; var reduceKeys = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); var documentsWrapped = documents.Select(doc => { var documentId = doc.__document_id; foreach (var reduceKey in actions.MappedResults.DeleteMappedResultsForDocumentId((string)documentId, name)) { reduceKeys.Add(reduceKey); } return(doc); }); foreach (var doc in RobustEnumerationIndex(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc); var docIdValue = documentIdFetcher(doc); if (docIdValue == null) { throw new InvalidOperationException("Could not find document id for this document"); } var reduceValue = viewGenerator.GroupByExtraction(doc); if (reduceValue == null) { logIndexing.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue); continue; } var reduceKey = ReduceKeyToString(reduceValue); var docId = docIdValue.ToString(); reduceKeys.Add(reduceKey); var data = GetMapedData(doc); logIndexing.DebugFormat("Mapped result for '{0}': '{1}'", name, data); var hash = ComputeHash(name, reduceKey); actions.MappedResults.PutMappedResult(name, docId, reduceKey, data, hash); actions.Indexing.IncrementSuccessIndexing(); } if (reduceKeys.Count > 0) { actions.Tasks.AddTask(new ReduceTask { Index = name, ReduceKeys = reduceKeys.ToArray() }, minimumTimestamp); } logIndexing.DebugFormat("Mapped {0} documents for {1}", count, name); }
protected AbstractIndexingExecuter(WorkContext context) { this.transactionalStorage = context.TransactionalStorage; this.context = context; this.scheduler = context.TaskScheduler; }
public ReducingExecuter(ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler) : base(transactionalStorage, context, scheduler) { }
public abstract void Remove(string[] keys, WorkContext context);
public IndexingExecuter(WorkContext context) : base(context) { autoTuner = new IndexBatchSizeAutoTuner(context); }
public override void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, (indexWriter, analyzer) => { bool madeChanges = false; PropertyDescriptorCollection properties = null; var processedKeys = new HashSet <string>(); var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); var documentsWrapped = documents.Select((dynamic doc) => { string documentId = doc.__document_id.ToString(); if (processedKeys.Add(documentId) == false) { return(doc); } madeChanges = true; batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, documentId); context.AddError(name, documentId, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, documentId)); indexWriter.DeleteDocuments(new Term(Constants.DocumentIdFieldName, documentId.ToLowerInvariant())); return(doc); }); foreach (var doc in RobustEnumerationIndex(documentsWrapped, viewGenerator.MapDefinition, actions, context)) { count++; IndexingResult indexingResult; if (doc is DynamicJsonObject) { indexingResult = ExtractIndexDataFromDocument((DynamicJsonObject)doc); } else { indexingResult = ExtractIndexDataFromDocument(properties, doc); } if (indexingResult.NewDocId != null && indexingResult.ShouldSkip == false) { var luceneDoc = new Document(); luceneDoc.Add(new Field(Constants.DocumentIdFieldName, indexingResult.NewDocId.ToLowerInvariant(), Field.Store.YES, Field.Index.NOT_ANALYZED)); madeChanges = true; CopyFieldsToDocument(luceneDoc, indexingResult.Fields); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, indexingResult.NewDocId); context.AddError(name, indexingResult.NewDocId, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, indexingResult.NewDocId, luceneDoc)); logIndexing.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); } actions.Indexing.IncrementSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return(madeChanges); }); logIndexing.DebugFormat("Indexed {0} documents for {1}", count, name); }
public TasksExecuter(ITransactionalStorage transactionalStorage, WorkContext context) { this.transactionalStorage = transactionalStorage; this.context = context; }
public ReduceDocuments(MapReduceIndex parent, AbstractViewGenerator viewGenerator, IEnumerable <IGrouping <int, object> > mappedResultsByBucket, int level, WorkContext context, IStorageActionsAccessor actions, HashSet <string> reduceKeys, int inputCount) { this.parent = parent; this.inputCount = inputCount; indexId = this.parent.indexId; ViewGenerator = viewGenerator; MappedResultsByBucket = mappedResultsByBucket; Level = level; Context = context; Actions = actions; ReduceKeys = reduceKeys; anonymousObjectToLuceneDocumentConverter = new AnonymousObjectToLuceneDocumentConverter(this.parent.context.Database, this.parent.indexDefinition, ViewGenerator, logIndexing); if (Level == 2) { batchers = Context.IndexUpdateTriggers.Select(x => x.CreateBatcher(indexId)) .Where(x => x != null) .ToList(); } }
protected AbstractIndexingExecuter(ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler) { this.transactionalStorage = transactionalStorage; this.context = context; this.scheduler = scheduler; }
protected IEnumerable <object> RobustEnumerationReduce(IEnumerable <object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { // not strictly accurate, but if we get that many errors, probably an error anyway. return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = actions.Indexing.IncrementReduceIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementReduceIndexingAttempt, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnFormat(exception, "Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)); try { actions.Indexing.IncrementReduceIndexingFailure(); } catch (Exception e) { // we don't care about error here, because it is an error on error problem logIndexing.WarnFormat(e, "Could not increment indexing failure rate for {0}", name); } } }.RobustEnumeration(input, func)); }
public IndependentBatchSizeAutoTuner(WorkContext context, PrefetchingUser user) : base(context) { this.User = user; InstallGauges(); }
public override void Remove(string[] keys, WorkContext context) { DeletionBatchInfo deletionBatchInfo = null; try { deletionBatchInfo = context.ReportDeletionBatchStarted(PublicName, keys.Length); context.TransactionalStorage.Batch(actions => { var storageCommitDuration = new Stopwatch(); actions.BeforeStorageCommit += storageCommitDuration.Start; actions.AfterStorageCommit += () => { storageCommitDuration.Stop(); deletionBatchInfo.PerformanceStats.Add(PerformanceStats.From(IndexingOperation.StorageCommit, storageCommitDuration.ElapsedMilliseconds)); }; var reduceKeyAndBuckets = new Dictionary <ReduceKeyAndBucket, int>(); var deleteMappedResultsDuration = new Stopwatch(); using (StopwatchScope.For(deleteMappedResultsDuration)) { if (actions.MapReduce.HasMappedResultsForIndex(indexId)) { foreach (var key in keys) { actions.MapReduce.DeleteMappedResultsForDocumentId(key, indexId, reduceKeyAndBuckets); context.CancellationToken.ThrowIfCancellationRequested(); } } } deletionBatchInfo.PerformanceStats.Add(PerformanceStats.From(IndexingOperation.Delete_DeleteMappedResultsForDocumentId, deleteMappedResultsDuration.ElapsedMilliseconds)); actions.MapReduce.UpdateRemovedMapReduceStats(indexId, reduceKeyAndBuckets, context.CancellationToken); var scheduleReductionsDuration = new Stopwatch(); using (StopwatchScope.For(scheduleReductionsDuration)) { foreach (var reduceKeyAndBucket in reduceKeyAndBuckets) { actions.MapReduce.ScheduleReductions(indexId, 0, reduceKeyAndBucket.Key); context.CancellationToken.ThrowIfCancellationRequested(); } } deletionBatchInfo.PerformanceStats.Add(PerformanceStats.From(IndexingOperation.Reduce_ScheduleReductions, scheduleReductionsDuration.ElapsedMilliseconds)); }); } finally { if (deletionBatchInfo != null) { context.ReportDeletionBatchCompleted(deletionBatchInfo); } } }
public RobustEnumerator(WorkContext context, int numberOfConsecutiveErrors) { this.context = context; this.numberOfConsecutiveErrors = numberOfConsecutiveErrors; }
public ReduceBatchSizeAutoTuner(WorkContext context) : base(context) { LastAmountOfItemsToRemember = 1; InstallGauges(); }
public IndexingExecuter(WorkContext context, Prefetcher prefetcher) : base(context) { autoTuner = new IndexBatchSizeAutoTuner(context); prefetchingBehavior = prefetcher.GetPrefetchingBehavior(PrefetchingUser.Indexer, autoTuner); }
public ReducingExecuter(WorkContext context, IndexReplacer indexReplacer) : base(context, indexReplacer) { autoTuner = new ReduceBatchSizeAutoTuner(context); }
protected void Write(WorkContext context, Func <IndexWriter, Analyzer, bool> action) { if (disposed) { throw new ObjectDisposedException("Index " + name + " has been disposed"); } lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List <Action>(); Analyzer analyzer = null; try { try { analyzer = CreateAnalyzer(new LowerCaseAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = new IndexWriter(directory, new StopAnalyzer(Version.LUCENE_29), IndexWriter.MaxFieldLength.UNLIMITED); } try { shouldRecreateSearcher = action(indexWriter, analyzer); foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocumented); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } WriteTempIndexToDiskIfNeeded(context); } finally { currentlyIndexDocumented.Clear(); if (analyzer != null) { analyzer.Close(); } foreach (Action dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) { RecreateSearcher(); } } }
public BaseBatchSizeAutoTuner(WorkContext context) { this.context = context; this.NumberOfItemsToIndexInSingleBatch = InitialNumberOfItems; }
protected IEnumerable <object> RobustEnumerationIndex(IEnumerable <object> input, IEnumerable <IndexingFunc> funcs, IStorageActionsAccessor actions, WorkContext context) { return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = actions.Indexing.IncrementIndexingAttempt, CancelMoveNext = actions.Indexing.DecrementIndexingAttempt, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); try { actions.Indexing.IncrementIndexingFailure(); } catch (Exception e) { // we don't care about error here, because it is an error on error problem logIndexing.WarnException( String.Format("Could not increment indexing failure rate for {0}", name), e); } } }.RobustEnumeration(input, funcs)); }
public void ReduceDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> mappedResults, WorkContext context, IStorageActionsAccessor actions, string[] reduceKeys) { actions.Indexing.SetCurrentIndexStatsTo(name); var count = 0; Write(context, (indexWriter, analyzer) => { var batchers = context.IndexUpdateTriggers.Select(x => x.CreateBatcher(name)) .Where(x => x != null) .ToList(); foreach (var reduceKey in reduceKeys) { var entryKey = reduceKey; indexWriter.DeleteDocuments(new Term("__reduce_key", entryKey.ToLowerInvariant())); batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryDeleted trigger for index '{0}', key: '{1}'", name, entryKey); context.AddError(name, entryKey, exception.Message ); }, trigger => trigger.OnIndexEntryDeleted(name, entryKey)); } PropertyDescriptorCollection properties = null; foreach (var doc in RobustEnumerationReduce(mappedResults, viewGenerator.ReduceDefinition, actions, context)) { count++; var fields = GetFields(doc, ref properties).ToList(); string reduceKeyAsString = ExtractReduceKey(viewGenerator, doc); var luceneDoc = new Document(); luceneDoc.Add(new Field("__reduce_key", reduceKeyAsString.ToLowerInvariant(), Field.Store.NO, Field.Index.NOT_ANALYZED)); foreach (var field in fields) { luceneDoc.Add(field); } batchers.ApplyAndIgnoreAllErrors( exception => { logIndexing.WarnFormat(exception, "Error when executed OnIndexEntryCreated trigger for index '{0}', key: '{1}'", name, reduceKeyAsString); context.AddError(name, reduceKeyAsString, exception.Message ); }, trigger => trigger.OnIndexEntryCreated(name, reduceKeyAsString, luceneDoc)); logIndexing.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKeyAsString, name, luceneDoc); AddDocumentToIndex(indexWriter, luceneDoc, analyzer); actions.Indexing.IncrementReduceSuccessIndexing(); } batchers.ApplyAndIgnoreAllErrors( e => { logIndexing.Warn("Failed to dispose on index update trigger", e); context.AddError(name, null, e.Message); }, x => x.Dispose()); return(true); }); if (logIndexing.IsDebugEnabled) { logIndexing.DebugFormat("Reduce resulted in {0} entries for {1} for reduce keys: {2}", count, name, string.Join(", ", reduceKeys)); } }
public abstract void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable <object> documents, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp);
public ReducingExecuter(WorkContext context) : base(context) { autoTuner = new ReduceBatchSizeAutoTuner(context); }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IndexingBatch batch, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp) { var count = 0; var sourceCount = 0; var sw = Stopwatch.StartNew(); var start = SystemTime.UtcNow; var changed = new HashSet <ReduceKeyAndBucket>(); var documentsWrapped = batch.Docs.Select(doc => { sourceCount++; var documentId = doc.__document_id; actions.MapReduce.DeleteMappedResultsForDocumentId((string)documentId, name, changed); return(doc); }) .Where(x => x is FilteredDocument == false); var stats = new IndexingWorkStats(); foreach ( var mappedResultFromDocument in GroupByDocumentId(context, RobustEnumerationIndex(documentsWrapped.GetEnumerator(), viewGenerator.MapDefinitions, actions, stats))) { var dynamicResults = mappedResultFromDocument.Select(x => (object)new DynamicJsonObject(RavenJObject.FromObject(x, jsonSerializer))).ToList(); foreach ( var doc in RobustEnumerationReduceDuringMapPhase(dynamicResults.GetEnumerator(), viewGenerator.ReduceDefinition, actions, context)) { count++; var reduceValue = viewGenerator.GroupByExtraction(doc); if (reduceValue == null) { logIndexing.Debug("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, mappedResultFromDocument.Key); continue; } var reduceKey = ReduceKeyToString(reduceValue); var docId = mappedResultFromDocument.Key.ToString(); var data = GetMappedData(doc); logIndexing.Debug("Mapped result for index '{0}' doc '{1}': '{2}'", name, docId, data); actions.MapReduce.PutMappedResult(name, docId, reduceKey, data); changed.Add(new ReduceKeyAndBucket(IndexingUtil.MapBucket(docId), reduceKey)); } } UpdateIndexingStats(context, stats); actions.MapReduce.ScheduleReductions(name, 0, changed); AddindexingPerformanceStat(new IndexingPerformanceStats { OutputCount = count, InputCount = sourceCount, Operation = "Map", Duration = sw.Elapsed, Started = start }); logIndexing.Debug("Mapped {0} documents for {1}", count, name); }
protected IEnumerable <object> RobustEnumerationIndex(IEnumerable <object> input, IEnumerable <IndexingFunc> funcs, IStorageActionsAccessor actions, WorkContext context, IndexingWorkStats stats) { return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = () => stats.IndexingAttempts++, CancelMoveNext = () => stats.IndexingAttempts--, OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); stats.IndexingErrors++; } }.RobustEnumeration(input, funcs)); }
protected void Write(WorkContext context, Func <IndexWriter, Analyzer, IndexingWorkStats, int> action) { if (disposed) { throw new ObjectDisposedException("Index " + name + " has been disposed"); } lock (writeLock) { bool shouldRecreateSearcher; var toDispose = new List <Action>(); Analyzer searchAnalyzer = null; try { try { searchAnalyzer = CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose); } catch (Exception e) { context.AddError(name, "Creating Analyzer", e.ToString()); throw; } if (indexWriter == null) { indexWriter = CreateIndexWriter(directory); } var stats = new IndexingWorkStats(); try { var changedDocs = action(indexWriter, searchAnalyzer, stats); docCountSinceLastOptimization += changedDocs; shouldRecreateSearcher = changedDocs > 0; foreach (IIndexExtension indexExtension in indexExtensions.Values) { indexExtension.OnDocumentsIndexed(currentlyIndexDocuments); } } catch (Exception e) { context.AddError(name, null, e.ToString()); throw; } UpdateIndexingStats(context, stats); WriteTempIndexToDiskIfNeeded(context); if (configuration.TransactionMode == TransactionMode.Safe) { Flush(); // just make sure changes are flushed to disk } } finally { currentlyIndexDocuments.Clear(); if (searchAnalyzer != null) { searchAnalyzer.Close(); } foreach (Action dispose in toDispose) { dispose(); } } if (shouldRecreateSearcher) { RecreateSearcher(); } } }
public IndexSearcherHolder(int indexId, WorkContext context) { this.indexId = indexId; this.context = context; }
// we don't care about tracking map/reduce stats here, since it is merely // an optimization step protected IEnumerable <object> RobustEnumerationReduceDuringMapPhase(IEnumerable <object> input, IndexingFunc func, IStorageActionsAccessor actions, WorkContext context) { // not strictly accurate, but if we get that many errors, probably an error anyway. return(new RobustEnumerator(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) { BeforeMoveNext = () => { }, // don't care CancelMoveNext = () => { }, // don't care OnError = (exception, o) => { context.AddError(name, TryGetDocKey(o), exception.Message ); logIndexing.WarnException( String.Format("Failed to execute indexing function on {0} on {1}", name, TryGetDocKey(o)), exception); } }.RobustEnumeration(input, func)); }