public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom)
{
log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom);
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return false; // index was deleted, probably
var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom)
.Where(x => x != null)
.Take(10000) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
if(jsonDocs.Length == 0)
return false;
var dateTime = jsonDocs.Select(x=>x.LastModified).Min();
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever.ProcessReadVetoes(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
return true;
}
catch (Exception e)
{
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
return false;
}
finally
{
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
var last = jsonDocs.Last();
actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified);
}
}