public void ReduceDocuments(AbstractViewGenerator viewGenerator,
IEnumerable <object> mappedResults,
WorkContext context,
DocumentStorageActions actions,
string reduceKey)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
indexWriter.DeleteDocuments(new Term("__reduce_key", reduceKey));
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, reduceKey));
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context))
{
count++;
var fields = GetFields(doc, ref properties);
var luceneDoc = new Document();
luceneDoc.Add(new Field("__reduce_key", reduceKey, Field.Store.NO, Field.Index.NOT_ANALYZED));
foreach (var field in fields)
{
luceneDoc.Add(field);
}
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, reduceKey, luceneDoc));
log.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKey, name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
actions.IncrementSuccessIndexing();
}
return(true);
});
log.DebugFormat("Reduce resulted in {0} entries for {1} for reduce key {2}", count, name, reduceKey);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Func<object, object> documentIdFetcher = null;
var reduceKeys = new HashSet<string>();
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc);
var docIdValue = documentIdFetcher(doc);
if (docIdValue == null)
throw new InvalidOperationException("Could not find document id for this document");
var reduceValue = viewGenerator.GroupByExtraction(doc);
if (reduceValue == null)
{
log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue);
continue;
}
var reduceKey = ReduceKeyToString(reduceValue);
var docId = docIdValue.ToString();
reduceKeys.Add(reduceKey);
string data = GetMapedData(doc);
log.DebugFormat("Mapped result for '{0}': '{1}'", name, data);
var hash = ComputeHash(name, reduceKey);
actions.PutMappedResult(name, docId, reduceKey, data, hash);
actions.IncrementSuccessIndexing();
}
foreach (var reduceKey in reduceKeys)
{
actions.AddTask(new ReduceTask
{
Index = name,
ReduceKey = reduceKey
});
}
log.DebugFormat("Mapped {0} documents for {1}", count, name);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable <object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
string currentId = null;
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
string newDocId;
IEnumerable <AbstractField> fields;
if (doc is DynamicJsonObject)
{
fields = ExtractIndexDataFromDocument((DynamicJsonObject)doc, out newDocId);
}
else
{
fields = ExtractIndexDataFromDocument(properties, doc, out newDocId);
}
if (currentId != newDocId) // new document id, so delete all old values matching it
{
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, newDocId));
indexWriter.DeleteDocuments(new Term("__document_id", newDocId));
}
if (newDocId != null)
{
var luceneDoc = new Document();
luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.NOT_ANALYZED));
currentId = newDocId;
CopyFieldsToDocumentButRemoveDuplicateValues(luceneDoc, fields);
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, newDocId, luceneDoc));
log.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
}
actions.IncrementSuccessIndexing();
}
return(currentId != null);
});
log.DebugFormat("Indexed {0} documents for {1}", count, name);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
string currentId = null;
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
string newDocId;
IEnumerable<AbstractField> fields;
if (doc is DynamicJsonObject)
fields = ExtractIndexDataFromDocument((DynamicJsonObject) doc, out newDocId);
else
fields = ExtractIndexDataFromDocument(properties, doc, out newDocId);
if (currentId != newDocId) // new document id, so delete all old values matching it
{
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, newDocId));
indexWriter.DeleteDocuments(new Term("__document_id", newDocId));
}
if (newDocId != null)
{
var luceneDoc = new Document();
luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.NOT_ANALYZED));
currentId = newDocId;
CopyFieldsToDocumentButRemoveDuplicateValues(luceneDoc, fields);
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, newDocId, luceneDoc));
log.DebugFormat("Index '{0}' resulted in: {1}", name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
}
actions.IncrementSuccessIndexing();
}
return currentId != null;
});
log.DebugFormat("Indexed {0} documents for {1}", count, name);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable <dynamic> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Func <object, object> documentIdFetcher = null;
var reduceKeys = new HashSet <string>();
var documentsWrapped = documents.Select(doc =>
{
var documentId = doc.__document_id;
foreach (var reduceKey in actions.DeleteMappedResultsForDocumentId((string)documentId, name))
{
reduceKeys.Add(reduceKey);
}
return(doc);
});
foreach (var doc in RobustEnumeration(documentsWrapped, viewGenerator.MapDefinition, actions, context))
{
count++;
documentIdFetcher = CreateDocumentIdFetcherIfNeeded(documentIdFetcher, doc);
var docIdValue = documentIdFetcher(doc);
if (docIdValue == null)
{
throw new InvalidOperationException("Could not find document id for this document");
}
var reduceValue = viewGenerator.GroupByExtraction(doc);
if (reduceValue == null)
{
log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByExtraction, docIdValue);
continue;
}
var reduceKey = ReduceKeyToString(reduceValue);
var docId = docIdValue.ToString();
reduceKeys.Add(reduceKey);
string data = GetMapedData(doc);
log.DebugFormat("Mapped result for '{0}': '{1}'", name, data);
var hash = ComputeHash(name, reduceKey);
actions.PutMappedResult(name, docId, reduceKey, data, hash);
actions.IncrementSuccessIndexing();
}
foreach (var reduceKey in reduceKeys)
{
actions.AddTask(new ReduceTask
{
Index = name,
ReduceKey = reduceKey
});
}
log.DebugFormat("Mapped {0} documents for {1}", count, name);
}
public void ReduceDocuments(AbstractViewGenerator viewGenerator,
IEnumerable<object> mappedResults,
WorkContext context,
DocumentStorageActions actions,
string reduceKey)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
indexWriter.DeleteDocuments(new Term("__reduce_key", reduceKey));
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryDeleted(name, reduceKey));
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context))
{
count++;
var fields = GetFields(doc, ref properties);
var luceneDoc = new Document();
luceneDoc.Add(new Field("__reduce_key", reduceKey, Field.Store.NO, Field.Index.NOT_ANALYZED));
foreach (var field in fields)
{
luceneDoc.Add(field);
}
context.IndexUpdateTriggers.Apply(trigger => trigger.OnIndexEntryCreated(name, reduceKey, luceneDoc));
log.DebugFormat("Reduce key {0} result in index {1} gave document: {2}", reduceKey, name, luceneDoc);
indexWriter.AddDocument(luceneDoc);
actions.IncrementSuccessIndexing();
}
return true;
});
log.DebugFormat("Reduce resulted in {0} entries for {1} for reduce key {2}", count, name, reduceKey);
}
