public void Execute(DocumentDatabase database)
{
Database = database;
Database.OnDocumentChange += (sender, notification) =>
{
if (notification.Id == null)
return;
if (!notification.Id.StartsWith("Raven/SqlReplication/Configuration/", StringComparison.InvariantCultureIgnoreCase))
return;
replicationConfigs = null;
statistics.Clear();
};
GetReplicationStatus();
prefetchingBehavior = new PrefetchingBehavior(Database.WorkContext, new IndexBatchSizeAutoTuner(Database.WorkContext));
var task = Task.Factory.StartNew(() =>
{
using (LogContext.WithDatabase(database.Name))
{
try
{
BackgroundSqlReplication();
}
catch (Exception e)
{
log.ErrorException("Fatal failure when replicating to SQL. All SQL Replication activity STOPPED", e);
}
}
}, TaskCreationOptions.LongRunning);
database.ExtensionsState.GetOrAdd(typeof(SqlReplicationTask).FullName, k => new DisposableAction(task.Wait));
}
private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs, IndexingBatchInfo indexingBatchInfo)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).OrderByDescending(x => x.Index.LastQueryTime).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
index =>
{
using (LogContext.WithDatabase(context.DatabaseName))
{
var performance = HandleIndexingFor(index, lastEtag, lastModified, CancellationToken.None);
if (performance != null)
{
indexingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performance);
}
}
});
return(lastEtag);
}
public IndexingExecuter(WorkContext context, DatabaseEtagSynchronizer synchronizer, Prefetcher prefetcher)
: base(context)
{
autoTuner = new IndexBatchSizeAutoTuner(context);
etagSynchronizer = synchronizer.GetSynchronizer(EtagSynchronizerType.Indexer);
prefetchingBehavior = prefetcher.GetPrefetchingBehavior(PrefetchingUser.Indexer);
}
public IndexingExecuter(WorkContext context, Prefetcher prefetcher)
: base(context)
{
autoTuner = new IndexBatchSizeAutoTuner(context);
this.prefetcher = prefetcher;
defaultPrefetchingBehavior = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
prefetchingBehaviors.TryAdd(defaultPrefetchingBehavior);
}
public IndexingExecuter(WorkContext context, Prefetcher prefetcher, IndexReplacer indexReplacer)
: base(context, indexReplacer)
{
autoTuner = new IndexBatchSizeAutoTuner(context);
this.prefetcher = prefetcher;
defaultPrefetchingBehavior = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
defaultPrefetchingBehavior.ShouldHandleUnusedDocumentsAddedAfterCommit = true;
prefetchingBehaviors.TryAdd(defaultPrefetchingBehavior);
}
private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.IndexedPerSecIncreaseBy(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).ToList();
ExecuteAllInterleaved(result, index => HandleIndexingFor(index, lastEtag, lastModified));
return(lastEtag);
}
private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
index => HandleIndexingFor(index, lastEtag, lastModified));
return(lastEtag);
}
public PreFetching()
{
store = NewDocumentStore();
var workContext = store.DocumentDatabase.WorkContext;
prefetchingBehavior = new PrefetchingBehavior(workContext, new IndexBatchSizeAutoTuner(workContext));
}
public IndexingExecuter(WorkContext context, Prefetcher prefetcher)
: base(context)
{
autoTuner = new IndexBatchSizeAutoTuner(context);
prefetchingBehavior = prefetcher.GetPrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
}
private bool GenerateIndexingBatchesAndPrefetchDocuments(List<IndexingGroup> groupedIndexes, ConcurrentDictionary<IndexingBatchOperation, object> indexBatchOperations)
{
bool operationWasCancelled = false;
context.Database.MappingThreadPool.ExecuteBatch(groupedIndexes,
indexingGroup =>
{
bool operationAdded = false;
try
{
indexingGroup.PrefetchDocuments();
var curGroupJsonDocs = indexingGroup.JsonDocs;
if (Log.IsDebugEnabled)
{
Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
curGroupJsonDocs.Count, indexingGroup.LastIndexedEtag, string.Join(", ", curGroupJsonDocs.Select(x => x.Key)));
}
indexingGroup.BatchInfo =
context.ReportIndexingBatchStarted(curGroupJsonDocs.Count,
curGroupJsonDocs.Sum(x => x.SerializedSizeOnDisk),
indexingGroup.Indexes.Select(x => x.Index.PublicName).ToList());
context.CancellationToken.ThrowIfCancellationRequested();
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(curGroupJsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
List<IndexToWorkOn> filteredOutIndexes;
var indexBatches = FilterIndexes(indexingGroup.Indexes, curGroupJsonDocs, lastEtag, out filteredOutIndexes).OrderByDescending(x => x.Index.LastQueryTime).ToList();
foreach (var filteredOutIndex in filteredOutIndexes)
{
indexingGroup.SignalIndexingComplete();
filteredOutIndex.Index.IsMapIndexingInProgress = false;
}
foreach (var indexBatch in indexBatches)
{
var indexingBatchOperation = new IndexingBatchOperation
{
IndexingBatch = indexBatch,
LastEtag = lastEtag,
LastModified = lastModified,
IndexingBatchInfo = indexingGroup.BatchInfo
};
if (indexBatchOperations.TryAdd(indexingBatchOperation, new object()))
{
indexingBatchOperation.IndexingBatch.OnIndexingComplete += indexingGroup.SignalIndexingComplete;
operationAdded = true;
}
}
}
catch (OperationCanceledException)
{
operationWasCancelled = true;
}
catch (InvalidDataException e)
{
Log.ErrorException("Failed to index because of data corruption. ", e);
indexingGroup.Indexes.ForEach(index =>
context.AddError(index.IndexId, index.Index.PublicName, null, e, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message)));
}
finally
{
if (operationAdded == false)
{
indexingGroup.Indexes.ForEach(x =>
{
indexingGroup.SignalIndexingComplete();
x.Index.IsMapIndexingInProgress = false;
});
}
}
}, description: string.Format("Prefetching Index Groups for {0} groups", groupedIndexes.Count));
return operationWasCancelled;
}
public IndexingExecuter(WorkContext context)
: base(context)
{
autoTuner = new IndexBatchSizeAutoTuner(context);
prefetchingBehavior = new PrefetchingBehavior(context, autoTuner);
}
public IndexingExecuter(WorkContext context)
: base(context)
{
autoTuner = new IndexBatchSizeAutoTuner(context);
prefetchingBehavior = new PrefetchingBehavior(context, autoTuner);
}