private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
return; // index was deleted, probably
}
var dateTime = jsonDocs.Min(x => x.LastModified);
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
{
return;
}
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
}
}
private IndexingPerformanceStats IndexDocuments(IStorageActionsAccessor actions, IndexingBatchForIndex indexingBatchForIndex, CancellationToken token)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexingBatchForIndex.IndexId);
if (viewGenerator == null)
{
return(null); // index was deleted, probably
}
var batch = indexingBatchForIndex.Batch;
IndexingPerformanceStats performanceStats = null;
try
{
if (Log.IsDebugEnabled)
{
string ids;
if (batch.Ids.Count < 256)
{
ids = string.Join(",", batch.Ids);
}
else
{
ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
}
Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, indexingBatchForIndex.Index.PublicName, ids);
}
context.CancellationToken.ThrowIfCancellationRequested();
token.ThrowIfCancellationRequested();
performanceStats = context.IndexStorage.Index(indexingBatchForIndex.IndexId, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue, token);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
{
return(null);
}
Log.WarnException(string.Format("Failed to index documents for index: {0}", indexingBatchForIndex.Index.PublicName), e);
context.AddError(indexingBatchForIndex.IndexId, indexingBatchForIndex.Index.PublicName, null, e);
}
return(performanceStats);
}
private void IndexDocuments(IStorageActionsAccessor actions, string index, IndexingBatch batch)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
return; // index was deleted, probably
}
try
{
if (log.IsDebugEnabled)
{
string ids;
if (batch.Ids.Count < 256)
{
ids = string.Join(",", batch.Ids);
}
else
{
ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
}
log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, index, ids);
}
context.CancellationToken.ThrowIfCancellationRequested();
context.IndexStorage.Index(index, viewGenerator, batch.Docs, context, actions, batch.DateTime ?? DateTime.MinValue);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
{
return;
}
log.WarnException(
string.Format("Failed to index documents for index: {0}", index),
e);
}
}
private void IndexDocuments(IStorageActionsAccessor actions, string index, IndexingBatch batch)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return; // index was deleted, probably
try
{
if (Log.IsDebugEnabled)
{
string ids;
if (batch.Ids.Count < 256)
ids = string.Join(",", batch.Ids);
else
{
ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
}
Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, index, ids);
}
context.CancellationToken.ThrowIfCancellationRequested();
context.IndexStorage.Index(index, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
return;
Log.WarnException(string.Format("Failed to index documents for index: {0}", index), e);
}
}
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return; // index was deleted, probably
var dateTime = jsonDocs.Min(x => x.LastModified) ?? DateTime.MinValue;
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
return;
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
}
}
private IndexingPerformanceStats IndexDocuments(IStorageActionsAccessor actions, IndexingBatchForIndex indexingBatchForIndex, CancellationToken token)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexingBatchForIndex.IndexId);
if (viewGenerator == null)
return null; // index was deleted, probably
var batch = indexingBatchForIndex.Batch;
IndexingPerformanceStats performanceStats = null;
try
{
if (Log.IsDebugEnabled)
{
string ids;
if (batch.Ids.Count < 256)
ids = string.Join(",", batch.Ids);
else
{
ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
}
Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, indexingBatchForIndex.Index.PublicName, ids);
}
context.CancellationToken.ThrowIfCancellationRequested();
token.ThrowIfCancellationRequested();
performanceStats = context.IndexStorage.Index(indexingBatchForIndex.IndexId, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue, token);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
return null;
Log.WarnException(string.Format("Failed to index documents for index: {0}", indexingBatchForIndex.Index.PublicName), e);
context.AddError(indexingBatchForIndex.IndexId, indexingBatchForIndex.Index.PublicName, null, e);
}
return performanceStats;
}
public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom)
{
log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom);
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return false; // index was deleted, probably
var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom)
.Where(x => x != null)
.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
if(jsonDocs.Length == 0)
return false;
var dateTime = jsonDocs.Select(x=>x.LastModified).Min();
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.EnsureIdInMetadata(doc)
.ProcessReadVetoes(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
return true;
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
return true;
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
return false;
}
finally
{
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
var last = jsonDocs.Last();
actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified);
}
}
