Beispiel #1
0
        private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
        {
            var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);

            if (viewGenerator == null)
            {
                return;                 // index was deleted, probably
            }
            var dateTime = jsonDocs.Min(x => x.LastModified);

            var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);

            try
            {
                log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
                context.IndexStorage.Index(index, viewGenerator,
                                           jsonDocs
                                           .Select(doc => documentRetriever
                                                   .ExecuteReadTriggers(doc, null, ReadOperation.Index))
                                           .Where(doc => doc != null)
                                           .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
            }
            catch (Exception e)
            {
                if (actions.IsWriteConflict(e))
                {
                    return;
                }
                log.WarnFormat(e, "Failed to index documents for index: {0}", index);
            }
        }
        private IndexingPerformanceStats IndexDocuments(IStorageActionsAccessor actions, IndexingBatchForIndex indexingBatchForIndex, CancellationToken token)
        {
            var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexingBatchForIndex.IndexId);

            if (viewGenerator == null)
            {
                return(null);                // index was deleted, probably
            }
            var batch = indexingBatchForIndex.Batch;

            IndexingPerformanceStats performanceStats = null;

            try
            {
                if (Log.IsDebugEnabled)
                {
                    string ids;
                    if (batch.Ids.Count < 256)
                    {
                        ids = string.Join(",", batch.Ids);
                    }
                    else
                    {
                        ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
                    }
                    Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, indexingBatchForIndex.Index.PublicName, ids);
                }
                context.CancellationToken.ThrowIfCancellationRequested();
                token.ThrowIfCancellationRequested();

                performanceStats = context.IndexStorage.Index(indexingBatchForIndex.IndexId, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue, token);
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (Exception e)
            {
                if (actions.IsWriteConflict(e))
                {
                    return(null);
                }

                Log.WarnException(string.Format("Failed to index documents for index: {0}", indexingBatchForIndex.Index.PublicName), e);
                context.AddError(indexingBatchForIndex.IndexId, indexingBatchForIndex.Index.PublicName, null, e);
            }

            return(performanceStats);
        }
Beispiel #3
0
        private void IndexDocuments(IStorageActionsAccessor actions, string index, IndexingBatch batch)
        {
            var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);

            if (viewGenerator == null)
            {
                return;                 // index was deleted, probably
            }
            try
            {
                if (log.IsDebugEnabled)
                {
                    string ids;
                    if (batch.Ids.Count < 256)
                    {
                        ids = string.Join(",", batch.Ids);
                    }
                    else
                    {
                        ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
                    }
                    log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, index, ids);
                }
                context.CancellationToken.ThrowIfCancellationRequested();

                context.IndexStorage.Index(index, viewGenerator, batch.Docs, context, actions, batch.DateTime ?? DateTime.MinValue);
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (Exception e)
            {
                if (actions.IsWriteConflict(e))
                {
                    return;
                }
                log.WarnException(
                    string.Format("Failed to index documents for index: {0}", index),
                    e);
            }
        }
Beispiel #4
0
		private void IndexDocuments(IStorageActionsAccessor actions, string index, IndexingBatch batch)
		{
			var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
			if (viewGenerator == null)
				return; // index was deleted, probably
			try
			{
				if (Log.IsDebugEnabled)
				{
					string ids;
					if (batch.Ids.Count < 256)
						ids = string.Join(",", batch.Ids);
					else
					{
						ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
					}
					Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, index, ids);
				}
				context.CancellationToken.ThrowIfCancellationRequested();

				context.IndexStorage.Index(index, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue);
			}
			catch (OperationCanceledException)
			{
				throw;
			}
			catch (Exception e)
			{
				if (actions.IsWriteConflict(e))
					return;
				Log.WarnException(string.Format("Failed to index documents for index: {0}", index), e);
			}
		}
Beispiel #5
0
        private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
        {
            var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
            if (viewGenerator == null)
                return; // index was deleted, probably

            var dateTime = jsonDocs.Min(x => x.LastModified) ?? DateTime.MinValue;

            var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
            try
            {
                log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
                context.IndexStorage.Index(index, viewGenerator,
                    jsonDocs
                    .Select(doc => documentRetriever
                        .ExecuteReadTriggers(doc, null, ReadOperation.Index))
                    .Where(doc => doc != null)
                    .Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
            }
            catch (Exception e)
            {
                if (actions.IsWriteConflict(e))
                    return;
                log.WarnFormat(e, "Failed to index documents for index: {0}", index);
            }
        }
Beispiel #6
0
		private IndexingPerformanceStats IndexDocuments(IStorageActionsAccessor actions, IndexingBatchForIndex indexingBatchForIndex, CancellationToken token)
		{
			var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexingBatchForIndex.IndexId);
			if (viewGenerator == null)
				return null; // index was deleted, probably

			var batch = indexingBatchForIndex.Batch;

			IndexingPerformanceStats performanceStats = null;
			try
			{
				if (Log.IsDebugEnabled)
				{
					string ids;
					if (batch.Ids.Count < 256)
						ids = string.Join(",", batch.Ids);
					else
					{
						ids = string.Join(", ", batch.Ids.Take(128)) + " ... " + string.Join(", ", batch.Ids.Skip(batch.Ids.Count - 128));
					}
					Log.Debug("Indexing {0} documents for index: {1}. ({2})", batch.Docs.Count, indexingBatchForIndex.Index.PublicName, ids);
				}
				context.CancellationToken.ThrowIfCancellationRequested();
				token.ThrowIfCancellationRequested();
				
				performanceStats = context.IndexStorage.Index(indexingBatchForIndex.IndexId, viewGenerator, batch, context, actions, batch.DateTime ?? DateTime.MinValue, token);
			}
			catch (OperationCanceledException)
			{
				throw;
			}
			catch (Exception e)
			{
				if (actions.IsWriteConflict(e))
					return null;

				Log.WarnException(string.Format("Failed to index documents for index: {0}", indexingBatchForIndex.Index.PublicName), e);
				context.AddError(indexingBatchForIndex.IndexId, indexingBatchForIndex.Index.PublicName, null, e);
			}

			return performanceStats;
		}
Beispiel #7
0
		public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom)
		{
			log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom);
			var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
			if (viewGenerator == null)
				return false; // index was deleted, probably

			var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom)
				.Where(x => x != null)
				.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
				.ToArray();

			if(jsonDocs.Length == 0)
				return false;

			var dateTime = jsonDocs.Select(x=>x.LastModified).Min();

			var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
			try
			{
				log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
				context.IndexStorage.Index(index, viewGenerator, 
					jsonDocs
					.Select(doc => documentRetriever
						.EnsureIdInMetadata(doc)
						.ProcessReadVetoes(doc, null, ReadOperation.Index))
					.Where(doc => doc != null)
					.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);

				return true;
			}
			catch (Exception e)
			{
				if (actions.IsWriteConflict(e))
					return true;
				log.WarnFormat(e, "Failed to index documents for index: {0}", index);
				return false;
			}
			finally
			{
				// whatever we succeeded in indexing or not, we have to update this
				// because otherwise we keep trying to re-index failed documents
				var last = jsonDocs.Last();
				actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified);
			}
			
		}