public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
PropertyDescriptor groupByPropertyDescriptor = null;
PropertyDescriptor documentIdPropertyDescriptor = null;
var reduceKeys = new HashSet<string>();
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
if (groupByPropertyDescriptor == null)
{
var props = TypeDescriptor.GetProperties(doc);
groupByPropertyDescriptor = props.Find(viewGenerator.GroupByField, false);
documentIdPropertyDescriptor = props.Find("__document_id", false);
}
var docIdValue = documentIdPropertyDescriptor.GetValue(doc);
if (docIdValue == null)
throw new InvalidOperationException("Could not find document id for this document");
var reduceValue = groupByPropertyDescriptor.GetValue(doc);
if (reduceValue == null)
{
log.DebugFormat("Field {0} is used as the reduce key and cannot be null, skipping document {1}", viewGenerator.GroupByField, docIdValue);
continue;
}
var reduceKey = reduceValue.ToString();
var docId = docIdValue.ToString();
reduceKeys.Add(reduceKey);
actions.PutMappedResult(name, docId, reduceKey, JObject.FromObject(doc).ToString(Formatting.None));
actions.IncrementSuccessIndexing();
}
foreach (var reduceKey in reduceKeys)
{
actions.AddTask(new ReduceTask
{
Index = name,
ReduceKey = reduceKey
});
}
log.DebugFormat("Mapped {0} documents for {1}", count, name);
}
public override void IndexDocuments(
AbstractViewGenerator viewGenerator,
IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
string currentId = null;
var converter = new AnonymousObjectToLuceneDocumentConverter();
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(documents, viewGenerator.MapDefinition, actions, context))
{
count++;
if (properties == null)
{
properties = TypeDescriptor.GetProperties(doc);
}
var newDocId = properties.Find("__document_id", false).GetValue(doc) as string;
var fields = converter.Index(doc, properties, indexDefinition);
if (currentId != newDocId) // new document id, so delete all old values matching it
{
indexWriter.DeleteDocuments(new Term("__document_id", newDocId));
}
if (newDocId != null)
{
var luceneDoc = new Document();
luceneDoc.Add(new Field("__document_id", newDocId, Field.Store.YES, Field.Index.UN_TOKENIZED));
currentId = newDocId;
CopyFieldsToDocumentButRemoveDuplicateValues(luceneDoc, fields);
log.DebugFormat("Indexing document {0}", luceneDoc);
indexWriter.AddDocument(luceneDoc);
}
actions.IncrementSuccessIndexing();
}
return currentId != null;
});
log.DebugFormat("Indexed {0} documents for {1}", count, name);
}
private static JsonDocument RetrieveDocument(DocumentStorageActions actions, IndexQueryResult queryResult,
HashSet<string> loadedIds)
{
if (queryResult.Projection == null)
{
if (loadedIds.Add(queryResult.Key))
return actions.DocumentByKey(queryResult.Key, null);
return null;
}
return new JsonDocument
{
Key = queryResult.Key,
DataAsJosn = queryResult.Projection,
};
}
public void ReduceDocuments(AbstractViewGenerator viewGenerator,
IEnumerable<object> mappedResults,
WorkContext context,
DocumentStorageActions actions,
string reduceKey)
{
actions.SetCurrentIndexStatsTo(name);
var count = 0;
Write(indexWriter =>
{
indexWriter.DeleteDocuments(new Term(viewGenerator.GroupByField, reduceKey));
var converter = new AnonymousObjectToLuceneDocumentConverter();
PropertyDescriptorCollection properties = null;
foreach (var doc in RobustEnumeration(mappedResults, viewGenerator.ReduceDefinition, actions, context))
{
count++;
if (properties == null)
{
properties = TypeDescriptor.GetProperties(doc);
}
var fields = converter.Index(doc, properties, indexDefinition);
var luceneDoc = new Document();
foreach (var field in fields)
{
luceneDoc.Add(field);
}
indexWriter.AddDocument(luceneDoc);
actions.IncrementSuccessIndexing();
}
return true;
});
log.DebugFormat("Reduce resulted in {0} entires for {1} for reduce key {2}", count, name, reduceKey);
}
protected IEnumerable<object> RobustEnumeration(IEnumerable<object> input, IndexingFunc func,
DocumentStorageActions actions, WorkContext context)
{
var wrapped = new StatefulEnumerableWrapper<dynamic>(input.GetEnumerator());
IEnumerator<object> en = func(wrapped).GetEnumerator();
do
{
var moveSuccessful = MoveNext(en, wrapped, context, actions);
if (moveSuccessful == false)
yield break;
if (moveSuccessful == true)
yield return en.Current;
else
en = func(wrapped).GetEnumerator();
} while (true);
}
public abstract void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents,
WorkContext context,
DocumentStorageActions actions);
private bool? MoveNext(IEnumerator en, StatefulEnumerableWrapper<object> innerEnumerator, WorkContext context,
DocumentStorageActions actions)
{
try
{
actions.IncrementIndexingAttempt();
var moveNext = en.MoveNext();
if (moveNext == false)
actions.DecrementIndexingAttempt();
return moveNext;
}
catch (Exception e)
{
actions.IncrementIndexingFailure();
context.AddError(name,
TryGetDocKey(innerEnumerator.Current),
e.Message
);
log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name,
GetDocId(innerEnumerator));
}
return null;
}
public void Batch(Action<DocumentStorageActions> action)
{
if (current.Value != null)
{
try
{
current.Value.PushTx();
action(current.Value);
}
finally
{
current.Value.PopTx();
}
return;
}
disposerLock.EnterReadLock();
try
{
using (var pht = new DocumentStorageActions(
instance,
database,
documentsColumns,
tasksColumns,
filesColumns,
indexStatsColumns,
mappedResultsColumns,
documentsModifiedByTransactionsColumns,
transactionsColumns,
identityColumns))
{
current.Value = pht;
action(pht);
if (pht.CommitCalled == false)
throw new InvalidOperationException("You forgot to call commit!");
onCommit();
}
}
finally
{
disposerLock.ExitReadLock();
current.Value = null;
}
}
public void Reduce(string index, AbstractViewGenerator viewGenerator, IEnumerable<object> mappedResults,
WorkContext context, DocumentStorageActions actions, string reduceKey)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index);
return;
}
var mapReduceIndex = value as MapReduceIndex;
if (mapReduceIndex == null)
{
log.WarnFormat("Tried to reduce on an index that is not a map/reduce index: {0}, ignoring", index);
return;
}
mapReduceIndex.ReduceDocuments(viewGenerator, mappedResults, context, actions, reduceKey);
}
public void Index(string index, AbstractViewGenerator viewGenerator, IEnumerable<dynamic> docs, WorkContext context,
DocumentStorageActions actions)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Tried to index on a non existant index {0}, ignoring", index);
return;
}
value.IndexDocuments(viewGenerator, docs, context, actions);
}