public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom) { log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom); var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) return false; // index was deleted, probably var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom) .Where(x => x != null) .Take(10000) // ensure that we won't go overboard with reading and blow up with OOM .ToArray(); if(jsonDocs.Length == 0) return false; var dateTime = jsonDocs.Select(x=>x.LastModified).Min(); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever.ProcessReadVetoes(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); return true; } catch (Exception e) { log.WarnFormat(e, "Failed to index documents for index: {0}", index); return false; } finally { // whatever we succeeded in indexing or not, we have to update this // because otherwise we keep trying to re-index failed documents var last = jsonDocs.Last(); actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified); } }