public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, DocumentStorageActions actions) { actions.SetCurrentIndexStatsTo(name); var count = 0; PropertyDescriptor groupByPropertyDescriptor = null; PropertyDescriptor documentIdPropertyDescriptor = null; var reduceKeys = new HashSet<string>(); foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context)) { count++; if (groupByPropertyDescriptor == null) { var props = TypeDescriptor.GetProperties(doc); groupByPropertyDescriptor = props.Find(viewGenerator.GroupByField, false); documentIdPropertyDescriptor = props.Find("__document_id", false); } var docIdValue = documentIdPropertyDescriptor.GetValue(doc); if (docIdValue == null) throw new InvalidOperationException("Could not find document id for this document"); var reduceValue = groupByPropertyDescriptor.GetValue(doc); if (reduceValue == null) { log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByField, docIdValue); continue; } var reduceKey = reduceValue.ToString(); var docId = docIdValue.ToString(); reduceKeys.Add(reduceKey); actions.PutMappedResult(name, docId, reduceKey, JObject.FromObject(doc).ToString(Formatting.None)); actions.IncrementSuccessIndexing(); } foreach (var reduceKey in reduceKeys) { actions.AddTask(new ReduceTask { Index = name, ReduceKey = reduceKey }); } log.DebugFormat("Mapped {0} documents for {1}", count, name); }
public override void IndexDocuments( AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, DocumentStorageActions actions) { actions.SetCurrentIndexStatsTo(name); var count = 0; Write(indexWriter => { string currentId = null; var converter = new AnonymousObjectToLuceneDocumentConverter(); PropertyDescriptorCollection properties = null; foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context)) { count++; if (properties == null) { properties = TypeDescriptor.GetProperties(doc); } var newDocId = properties.Find("__document_id", false).GetValue(doc) as string; var fields = converter.Index(doc, properties, indexDefinition); if (currentId != newDocId) // new document id, so delete all old values matching it { indexWriter.DeleteDocuments(new Term("__document_id", newDocId)); } if (newDocId != null) { var luceneDoc = new Document(); luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.UN_TOKENIZED)); currentId = newDocId; CopyFieldsToDocumentButRemoveDuplicateValues(luceneDoc, fields); log.DebugFormat("Indexing document {0}", luceneDoc); indexWriter.AddDocument(luceneDoc); } actions.IncrementSuccessIndexing(); } return currentId != null; }); log.DebugFormat("Indexed {0} documents for {1}", count, name); }
private static JsonDocument RetrieveDocument(DocumentStorageActions actions, IndexQueryResult queryResult, HashSet<string> loadedIds) { if (queryResult.Projection == null) { if (loadedIds.Add(queryResult.Key)) return actions.DocumentByKey(queryResult.Key, null); return null; } return new JsonDocument { Key = queryResult.Key, DataAsJosn = queryResult.Projection, }; }
public void ReduceDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> mappedResults, WorkContext context, DocumentStorageActions actions, string reduceKey) { actions.SetCurrentIndexStatsTo(name); var count = 0; Write(indexWriter => { indexWriter.DeleteDocuments(new Term(viewGenerator.GroupByField, reduceKey)); var converter = new AnonymousObjectToLuceneDocumentConverter(); PropertyDescriptorCollection properties = null; foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context)) { count++; if (properties == null) { properties = TypeDescriptor.GetProperties(doc); } var fields = converter.Index(doc, properties, indexDefinition); var luceneDoc = new Document(); foreach (var field in fields) { luceneDoc.Add(field); } indexWriter.AddDocument(luceneDoc); actions.IncrementSuccessIndexing(); } return true; }); log.DebugFormat("Reduce resulted in {0} entires for {1} for reduce key {2}", count, name, reduceKey); }
protected IEnumerable<object> RobustEnumeration(IEnumerable<object> input, IndexingFunc func, DocumentStorageActions actions, WorkContext context) { var wrapped = new StatefulEnumerableWrapper<dynamic>(input.GetEnumerator()); IEnumerator<object> en = func(wrapped).GetEnumerator(); do { var moveSuccessful = MoveNext(en, wrapped, context, actions); if (moveSuccessful == false) yield break; if (moveSuccessful == true) yield return en.Current; else en = func(wrapped).GetEnumerator(); } while (true); }
public abstract void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents, WorkContext context, DocumentStorageActions actions);
private bool? MoveNext(IEnumerator en, StatefulEnumerableWrapper<object> innerEnumerator, WorkContext context, DocumentStorageActions actions) { try { actions.IncrementIndexingAttempt(); var moveNext = en.MoveNext(); if (moveNext == false) actions.DecrementIndexingAttempt(); return moveNext; } catch (Exception e) { actions.IncrementIndexingFailure(); context.AddError(name, TryGetDocKey(innerEnumerator.Current), e.Message ); log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name, GetDocId(innerEnumerator)); } return null; }
public void Batch(Action<DocumentStorageActions> action) { if (current.Value != null) { try { current.Value.PushTx(); action(current.Value); } finally { current.Value.PopTx(); } return; } disposerLock.EnterReadLock(); try { using (var pht = new DocumentStorageActions( instance, database, documentsColumns, tasksColumns, filesColumns, indexStatsColumns, mappedResultsColumns, documentsModifiedByTransactionsColumns, transactionsColumns, identityColumns)) { current.Value = pht; action(pht); if (pht.CommitCalled == false) throw new InvalidOperationException("You forgot to call commit!"); onCommit(); } } finally { disposerLock.ExitReadLock(); current.Value = null; } }
public void Reduce(string index, AbstractViewGenerator viewGenerator, IEnumerable<object> mappedResults, WorkContext context, DocumentStorageActions actions, string reduceKey) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index); return; } var mapReduceIndex = value as MapReduceIndex; if (mapReduceIndex == null) { log.WarnFormat("Tried to reduce on an index that is not a map/reduce index: {0}, ignoring", index); return; } mapReduceIndex.ReduceDocuments(viewGenerator, mappedResults, context, actions, reduceKey); }
public void Index(string index, AbstractViewGenerator viewGenerator, IEnumerable<dynamic> docs, WorkContext context, DocumentStorageActions actions) { Index value; if (indexes.TryGetValue(index, out value) == false) { log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index); return; } value.IndexDocuments(viewGenerator, docs, context, actions); }