Ejemplo n.º 1
0
		public void Execute(DocumentDatabase database)
		{
			Database = database;
			Database.OnDocumentChange += (sender, notification) =>
			{
				if (notification.Id == null)
					return;
				if (!notification.Id.StartsWith("Raven/SqlReplication/Configuration/", StringComparison.InvariantCultureIgnoreCase))
					return;

				replicationConfigs = null;
				statistics.Clear();
			};

			GetReplicationStatus();

			prefetchingBehavior = new PrefetchingBehavior(Database.WorkContext, new IndexBatchSizeAutoTuner(Database.WorkContext));

			var task = Task.Factory.StartNew(() =>
			{
				using (LogContext.WithDatabase(database.Name))
				{
					try
					{
						BackgroundSqlReplication();
					}
					catch (Exception e)
					{
						log.ErrorException("Fatal failure when replicating to SQL. All SQL Replication activity STOPPED", e);
					}
				}
			}, TaskCreationOptions.LongRunning);
			database.ExtensionsState.GetOrAdd(typeof(SqlReplicationTask).FullName, k => new DisposableAction(task.Wait));
		}
Ejemplo n.º 2
0
        private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs, IndexingBatchInfo indexingBatchInfo)
        {
            var lastByEtag   = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
            var lastModified = lastByEtag.LastModified.Value;
            var lastEtag     = lastByEtag.Etag;

            context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);

            var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).OrderByDescending(x => x.Index.LastQueryTime).ToList();

            BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
                                                                  index =>
            {
                using (LogContext.WithDatabase(context.DatabaseName))
                {
                    var performance = HandleIndexingFor(index, lastEtag, lastModified, CancellationToken.None);

                    if (performance != null)
                    {
                        indexingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performance);
                    }
                }
            });

            return(lastEtag);
        }
Ejemplo n.º 3
0
 public IndexingExecuter(WorkContext context, DatabaseEtagSynchronizer synchronizer, Prefetcher prefetcher)
     : base(context)
 {
     autoTuner           = new IndexBatchSizeAutoTuner(context);
     etagSynchronizer    = synchronizer.GetSynchronizer(EtagSynchronizerType.Indexer);
     prefetchingBehavior = prefetcher.GetPrefetchingBehavior(PrefetchingUser.Indexer);
 }
Ejemplo n.º 4
0
 public IndexingExecuter(WorkContext context, Prefetcher prefetcher)
     : base(context)
 {
     autoTuner                  = new IndexBatchSizeAutoTuner(context);
     this.prefetcher            = prefetcher;
     defaultPrefetchingBehavior = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
     prefetchingBehaviors.TryAdd(defaultPrefetchingBehavior);
 }
Ejemplo n.º 5
0
 public IndexingExecuter(WorkContext context, Prefetcher prefetcher, IndexReplacer indexReplacer)
     : base(context, indexReplacer)
 {
     autoTuner                  = new IndexBatchSizeAutoTuner(context);
     this.prefetcher            = prefetcher;
     defaultPrefetchingBehavior = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
     defaultPrefetchingBehavior.ShouldHandleUnusedDocumentsAddedAfterCommit = true;
     prefetchingBehaviors.TryAdd(defaultPrefetchingBehavior);
 }
Ejemplo n.º 6
0
        private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
        {
            var lastByEtag   = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
            var lastModified = lastByEtag.LastModified.Value;
            var lastEtag     = lastByEtag.Etag;

            context.IndexedPerSecIncreaseBy(jsonDocs.Count);
            var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).ToList();

            ExecuteAllInterleaved(result, index => HandleIndexingFor(index, lastEtag, lastModified));

            return(lastEtag);
        }
Ejemplo n.º 7
0
        private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
        {
            var lastByEtag   = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
            var lastModified = lastByEtag.LastModified.Value;
            var lastEtag     = lastByEtag.Etag;

            context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);

            var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).ToList();

            BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
                                                                  index => HandleIndexingFor(index, lastEtag, lastModified));

            return(lastEtag);
        }
Ejemplo n.º 8
0
		public PreFetching()
		{
			store = NewDocumentStore();
				var workContext = store.DocumentDatabase.WorkContext;
			prefetchingBehavior = new PrefetchingBehavior(workContext, new IndexBatchSizeAutoTuner(workContext));
		}
Ejemplo n.º 9
0
 public IndexingExecuter(WorkContext context, Prefetcher prefetcher)
     : base(context)
 {
     autoTuner           = new IndexBatchSizeAutoTuner(context);
     prefetchingBehavior = prefetcher.GetPrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
 }
Ejemplo n.º 10
0
        private bool GenerateIndexingBatchesAndPrefetchDocuments(List<IndexingGroup> groupedIndexes, ConcurrentDictionary<IndexingBatchOperation, object> indexBatchOperations)
                    {
            bool operationWasCancelled = false;

            context.Database.MappingThreadPool.ExecuteBatch(groupedIndexes,
                indexingGroup =>
                {
                    bool operationAdded = false;
                        try
                        {
                        indexingGroup.PrefetchDocuments();
                        var curGroupJsonDocs = indexingGroup.JsonDocs;
                            if (Log.IsDebugEnabled)
                            {
                                Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
                                curGroupJsonDocs.Count, indexingGroup.LastIndexedEtag, string.Join(", ", curGroupJsonDocs.Select(x => x.Key)));
                            }

                        indexingGroup.BatchInfo =
                            context.ReportIndexingBatchStarted(curGroupJsonDocs.Count,
                                curGroupJsonDocs.Sum(x => x.SerializedSizeOnDisk),
                                indexingGroup.Indexes.Select(x => x.Index.PublicName).ToList());

                            context.CancellationToken.ThrowIfCancellationRequested();
                        var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(curGroupJsonDocs);
                        var lastModified = lastByEtag.LastModified.Value;
                        var lastEtag = lastByEtag.Etag;
                        List<IndexToWorkOn> filteredOutIndexes;
                        var indexBatches = FilterIndexes(indexingGroup.Indexes, curGroupJsonDocs, lastEtag, out filteredOutIndexes).OrderByDescending(x => x.Index.LastQueryTime).ToList();

                        foreach (var filteredOutIndex in filteredOutIndexes)
                            {
                            indexingGroup.SignalIndexingComplete();
                            filteredOutIndex.Index.IsMapIndexingInProgress = false;
                            }

                        foreach (var indexBatch in indexBatches)
                        {
                            var indexingBatchOperation = new IndexingBatchOperation
                            {
                                IndexingBatch = indexBatch,
                                LastEtag = lastEtag,
                                LastModified = lastModified,
                                IndexingBatchInfo = indexingGroup.BatchInfo
                            };

                            if (indexBatchOperations.TryAdd(indexingBatchOperation, new object()))
                        {
                                indexingBatchOperation.IndexingBatch.OnIndexingComplete += indexingGroup.SignalIndexingComplete;
                                operationAdded = true;
                        }
                        }
                    }
                        catch (OperationCanceledException)
                        {
                        operationWasCancelled = true;
                        }
                    catch (InvalidDataException e)
                        {
                        Log.ErrorException("Failed to index because of data corruption. ", e);
                        indexingGroup.Indexes.ForEach(index =>
                            context.AddError(index.IndexId, index.Index.PublicName, null, e, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message)));
                        }
                        finally
                        {
                        if (operationAdded == false)
                            {
                            indexingGroup.Indexes.ForEach(x =>
                            {
                                indexingGroup.SignalIndexingComplete();
                                x.Index.IsMapIndexingInProgress = false;
                            });
                            }
                        }
                }, description: string.Format("Prefetching Index Groups for {0} groups", groupedIndexes.Count));
            return operationWasCancelled;
                    }
Ejemplo n.º 11
0
 public IndexingExecuter(WorkContext context)
     : base(context)
 {
     autoTuner           = new IndexBatchSizeAutoTuner(context);
     prefetchingBehavior = new PrefetchingBehavior(context, autoTuner);
 }
Ejemplo n.º 12
0
		public IndexingExecuter(WorkContext context)
			: base(context)
		{
			autoTuner = new IndexBatchSizeAutoTuner(context);
			prefetchingBehavior = new PrefetchingBehavior(context, autoTuner);
		}