private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) { return; // index was deleted, probably } var dateTime = jsonDocs.Min(x => x.LastModified); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); } catch (Exception e) { if (actions.IsWriteConflict(e)) { return; } log.WarnFormat(e, "Failed to index documents for index: {0}", index); } }
private IndexingPerformanceStats IndexDocuments(IStorageActionsAccessor actions, IndexingBatchForIndex indexingBatchForIndex, CancellationToken token) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexingBatchForIndex.IndexId); if (viewGenerator == null) { return(null); // index was deleted, probably } var batch = indexingBatchForIndex.Batch; IndexingPerformanceStats performanceStats = null; try { if (Log.IsDebugEnabled) { string ids; if (batch.Ids.Count < 256) { ids = string.Join(",", batch.Ids); } else { ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128)); } Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, indexingBatchForIndex.Index.PublicName, ids); } context.CancellationToken.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested(); performanceStats = context.IndexStorage.Index(indexingBatchForIndex.IndexId, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue, token); } catch (OperationCanceledException) { throw; } catch (Exception e) { if (actions.IsWriteConflict(e)) { return(null); } Log.WarnException(string.Format("Failed to index documents for index: {0}", indexingBatchForIndex.Index.PublicName), e); context.AddError(indexingBatchForIndex.IndexId, indexingBatchForIndex.Index.PublicName, null, e); } return(performanceStats); }
private void IndexDocuments(IStorageActionsAccessor actions, string index, IndexingBatch batch) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) { return; // index was deleted, probably } try { if (log.IsDebugEnabled) { string ids; if (batch.Ids.Count < 256) { ids = string.Join(",", batch.Ids); } else { ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128)); } log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, index, ids); } context.CancellationToken.ThrowIfCancellationRequested(); context.IndexStorage.Index(index, viewGenerator, batch.Docs, context, actions, batch.DateTime ?? DateTime.MinValue); } catch (OperationCanceledException) { throw; } catch (Exception e) { if (actions.IsWriteConflict(e)) { return; } log.WarnException( string.Format("Failed to index documents for index: {0}", index), e); } }
private void IndexDocuments(IStorageActionsAccessor actions, string index, IndexingBatch batch) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) return; // index was deleted, probably try { if (Log.IsDebugEnabled) { string ids; if (batch.Ids.Count < 256) ids = string.Join(",", batch.Ids); else { ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128)); } Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, index, ids); } context.CancellationToken.ThrowIfCancellationRequested(); context.IndexStorage.Index(index, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue); } catch (OperationCanceledException) { throw; } catch (Exception e) { if (actions.IsWriteConflict(e)) return; Log.WarnException(string.Format("Failed to index documents for index: {0}", index), e); } }
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) return; // index was deleted, probably var dateTime = jsonDocs.Min(x => x.LastModified) ?? DateTime.MinValue; var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever .ExecuteReadTriggers(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); } catch (Exception e) { if (actions.IsWriteConflict(e)) return; log.WarnFormat(e, "Failed to index documents for index: {0}", index); } }
private IndexingPerformanceStats IndexDocuments(IStorageActionsAccessor actions, IndexingBatchForIndex indexingBatchForIndex, CancellationToken token) { var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexingBatchForIndex.IndexId); if (viewGenerator == null) return null; // index was deleted, probably var batch = indexingBatchForIndex.Batch; IndexingPerformanceStats performanceStats = null; try { if (Log.IsDebugEnabled) { string ids; if (batch.Ids.Count < 256) ids = string.Join(",", batch.Ids); else { ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128)); } Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, indexingBatchForIndex.Index.PublicName, ids); } context.CancellationToken.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested(); performanceStats = context.IndexStorage.Index(indexingBatchForIndex.IndexId, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue, token); } catch (OperationCanceledException) { throw; } catch (Exception e) { if (actions.IsWriteConflict(e)) return null; Log.WarnException(string.Format("Failed to index documents for index: {0}", indexingBatchForIndex.Index.PublicName), e); context.AddError(indexingBatchForIndex.IndexId, indexingBatchForIndex.Index.PublicName, null, e); } return performanceStats; }
public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom) { log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom); var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index); if (viewGenerator == null) return false; // index was deleted, probably var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom) .Where(x => x != null) .Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM .ToArray(); if(jsonDocs.Length == 0) return false; var dateTime = jsonDocs.Select(x=>x.LastModified).Min(); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); try { log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index); context.IndexStorage.Index(index, viewGenerator, jsonDocs .Select(doc => documentRetriever .EnsureIdInMetadata(doc) .ProcessReadVetoes(doc, null, ReadOperation.Index)) .Where(doc => doc != null) .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime); return true; } catch (Exception e) { if (actions.IsWriteConflict(e)) return true; log.WarnFormat(e, "Failed to index documents for index: {0}", index); return false; } finally { // whatever we succeeded in indexing or not, we have to update this // because otherwise we keep trying to re-index failed documents var last = jsonDocs.Last(); actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified); } }