public static void Export(BlittableJsonTextWriter writer, Index index, JsonOperationContext context, bool removeAnalyzers)
{
if (index.Type == IndexType.Faulty)
{
return;
}
writer.WriteStartObject();
writer.WritePropertyName(nameof(IndexDefinition.Type));
writer.WriteString(index.Type.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(IndexDefinition));
if (index.Type == IndexType.Map || index.Type == IndexType.MapReduce || index.Type == IndexType.JavaScriptMap || index.Type == IndexType.JavaScriptMapReduce)
{
var indexDefinition = index.GetIndexDefinition();
writer.WriteIndexDefinition(context, indexDefinition, removeAnalyzers);
}
else if (index.Type == IndexType.AutoMap || index.Type == IndexType.AutoMapReduce)
{
index.Definition.Persist(context, writer);
}
else
{
throw new NotSupportedException(index.Type.ToString());
}
writer.WriteEndObject();
}
protected Task <IOperationResult> ExecutePatch(IndexQueryServerSide query, Index index, QueryOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, QueryOperationContext queryContext, Action <DeterminateProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(query, index, options, queryContext, onProgress,
(key, retrieveDetails) =>
{
var command = new PatchDocumentCommand(queryContext.Documents, key,
expectedChangeVector: null,
skipPatchIfChangeVectorMismatch: false,
patch: (patch, patchArgs),
patchIfMissing: (null, null),
createIfMissing: null,
database: Database,
debugMode: false,
isTest: false,
collectResultsNeeded: true,
returnDocument: false);
return new BulkOperationCommand <PatchDocumentCommand>(command, retrieveDetails,
x => new BulkOperationResult.PatchDetails
{
Id = key,
ChangeVector = x.PatchResult.ChangeVector,
Status = x.PatchResult.Status
},
c => c.PatchResult?.Dispose());
}, token));
private static TimeSpan?GetLastQueryInfo(Index index, DateTime now)
{
TimeSpan?lastQueried = null;
var lastQueryingTime = index.GetLastQueryingTime();
if (lastQueryingTime.HasValue)
{
lastQueried = now - lastQueryingTime;
}
return(lastQueried);
}
protected Task <IOperationResult> ExecuteDelete(IndexQueryServerSide query, Index index, QueryOperationOptions options, QueryOperationContext queryContext, Action <DeterminateProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(query, index, options, queryContext, onProgress, (key, retrieveDetails) =>
{
var command = new DeleteDocumentCommand(key, null, Database);
return new BulkOperationCommand <DeleteDocumentCommand>(command, retrieveDetails, x => new BulkOperationResult.DeleteDetails
{
Id = key,
Etag = x.DeleteResult?.Etag
}, null);
}, token));
}
public static DynamicQueryMapping Create(Index index)
{
if (index.Type.IsAuto() == false)
{
throw new InvalidOperationException();
}
var mapping = new DynamicQueryMapping
{
ForCollection = index.Collections.First(),
MapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal)
};
var definition = (AutoIndexDefinitionBase)index.Definition;
foreach (var field in definition.MapFields)
{
var autoField = (AutoIndexField)field.Value;
mapping.MapFields[field.Key] = DynamicQueryMappingItem.Create(
new QueryFieldName(autoField.Name, autoField.HasQuotedName),
autoField.Aggregation,
isFullTextSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Search),
isExactSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Exact),
hasHighlighting: autoField.Indexing.HasFlag(AutoFieldIndexing.Highlighting),
hasSuggestions: autoField.HasSuggestions,
spatial: autoField.Spatial);
}
if (index.Type.IsMapReduce())
{
mapping.IsGroupBy = true;
var mapReduceDefinition = (AutoMapReduceIndexDefinition)definition;
mapping.GroupByFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal);
foreach (var field in mapReduceDefinition.GroupByFields)
{
var autoField = field.Value;
mapping.GroupByFields[field.Key] = DynamicQueryMappingItem.CreateGroupBy(
new QueryFieldName(autoField.Name, autoField.HasQuotedName),
autoField.GroupByArrayBehavior,
isSpecifiedInWhere: true,
isFullTextSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Search),
isExactSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Exact));
}
}
return(mapping);
}
private static void FillIndexInfo(Index index, DocumentsOperationContext context, DateTime now, DatabaseStatusReport report)
{
var stats = index.GetIndexStats(context);
var lastQueried = GetLastQueryInfo(index, now);
//We might have old version of this index with the same name
report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
LastQueried = lastQueried,
IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase),
IsStale = stats.IsStale,
State = index.State,
LastTransactionId = index.LastTransactionId
};
}
private bool TryMatchExistingIndexToQuery(DynamicQueryMapping map, out Index index)
{
var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore);
var matchResult = dynamicQueryToIndex.Match(map);
switch (matchResult.MatchType)
{
case DynamicQueryMatchType.Complete:
case DynamicQueryMatchType.CompleteButIdle:
index = _indexStore.GetIndex(matchResult.IndexName);
if (index == null)
{
// the auto index was deleted
break;
}
return(true);
case DynamicQueryMatchType.Partial:
// At this point, we found an index that has some fields we need and
// isn't incompatible with anything else we're asking for
// We need to clone that other index
// We need to add all our requested indexes information to our cloned index
// We can then use our new index instead
var currentIndex = _indexStore.GetIndex(matchResult.IndexName);
if (currentIndex != null)
{
if (map.SupersededIndexes == null)
{
map.SupersededIndexes = new List <Index>();
}
map.SupersededIndexes.Add(currentIndex);
map.ExtendMappingBasedOn((AutoIndexDefinitionBase)currentIndex.Definition);
}
break;
}
index = null;
return(false);
}
protected async Task <SuggestionQueryResult> ExecuteSuggestion(
IndexQueryServerSide query,
Index index,
QueryOperationContext queryContext,
long?existingResultEtag,
OperationCancelToken token)
{
if (query.Metadata.SelectFields.Length == 0)
{
throw new InvalidQueryException("Suggestion query must have at least one suggest token in SELECT.", query.Metadata.QueryText, query.QueryParameters);
}
var fields = index.Definition.IndexFields;
foreach (var f in query.Metadata.SelectFields)
{
if (f.IsSuggest == false)
{
throw new InvalidQueryException("Suggestion query must have only suggest tokens in SELECT.", query.Metadata.QueryText, query.QueryParameters);
}
var selectField = (SuggestionField)f;
if (fields.TryGetValue(selectField.Name, out var field) == false)
{
throw new InvalidOperationException($"Index '{index.Name}' does not have a field '{selectField.Name}'.");
}
if (field.HasSuggestions == false)
{
throw new InvalidOperationException($"Index '{index.Name}' does not have suggestions configured for field '{selectField.Name}'.");
}
}
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag(queryContext, query.Metadata);
if (etag == existingResultEtag.Value)
{
return(SuggestionQueryResult.NotModifiedResult);
}
}
return(await index.SuggestionQuery(query, queryContext, token));
}
private IndexMetrics GetIndexMetrics(Index index)
{
var result = new IndexMetrics();
result.IndexName = index.Name;
result.Priority = index.Definition.Priority;
result.State = index.State;
result.Errors = (int)index.GetErrorCount();
var stats = index.GetStats();
if (stats.LastQueryingTime.HasValue)
{
var lastQueryingTime = stats.LastQueryingTime.Value;
result.TimeSinceLastQueryInSec = (SystemTime.UtcNow - lastQueryingTime).TotalSeconds;
}
if (stats.LastIndexingTime.HasValue)
{
var lastIndexingType = stats.LastIndexingTime.Value;
result.TimeSinceLastIndexingInSec = (SystemTime.UtcNow - lastIndexingType).TotalSeconds;
}
result.LockMode = index.Definition.LockMode;
result.IsInvalid = stats.IsInvalidIndex;
result.Status = index.Status;
result.MappedPerSec = index.MapsPerSec?.OneMinuteRate ?? 0;
result.ReducedPerSec = index.ReducesPerSec?.OneMinuteRate ?? 0;
result.Type = index.Type;
result.EntriesCount = stats.EntriesCount;
return(result);
}
private static void ActualTest(int numberOfUsers, string[] locations, Index index,
MapReduceIndexingContext mapReduceContext, IIndexingWork reducer, DocumentDatabase database, string outputToCollectionName)
{
using (index._contextPool.AllocateOperationContext(out TransactionOperationContext indexContext))
{
ulong hashOfReduceKey = 73493;
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < numberOfUsers; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 1,
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected, results[i].Data["Count"]);
}
}
// update
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 2, // increased by 1
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
tx.Commit();
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected + 1, results[i].Data["Count"]);
}
}
// delete
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
store.Delete(i);
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected - 1, results[i].Data["Count"]);
}
}
// delete entries for one reduce key
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < numberOfUsers; i++)
{
if (i % locations.Length == 0)
{
store.Delete(i);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length - 1, results.Count);
}
}
}
private static async Task ActualTest(int numberOfUsers, string[] locations, Index index,
MapReduceIndexingContext mapReduceContext, IIndexingWork reducer, DocumentDatabase database)
{
TransactionOperationContext indexContext;
using (index._contextPool.AllocateOperationContext(out indexContext))
{
ulong hashOfReduceKey = 73493;
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < numberOfUsers; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 1,
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
using (var termSingleUse = QueryOperationContext.ShortTermSingleUse(database))
{
var queryResult = await
index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"),
termSingleUse,
OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected, results[i].Data["Count"]);
}
}
// update
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 2, // increased by 1
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
tx.Commit();
}
using (var shortTermSingleUse = QueryOperationContext.ShortTermSingleUse(database))
{
var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"),
shortTermSingleUse,
OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected + 1, results[i].Data["Count"]);
}
}
// delete
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
store.Delete(i);
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
tx.Commit();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
using (var documentsOperationContext = QueryOperationContext.ShortTermSingleUse(database))
{
var queryResult = await index.Query(new IndexQueryServerSide("FROM Users ORDER BY Location"),
documentsOperationContext,
OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected - 1, results[i].Data["Count"]);
}
}
}
}
private async Task CleanupSupersededAutoIndexes(Index index, DynamicQueryMapping map, string raftRequestId, CancellationToken token)
{
if (map.SupersededIndexes == null || map.SupersededIndexes.Count == 0)
{
return;
}
// this is meant to remove superseded indexes immediately when they are of no use
// however, they'll also be cleaned by the idle timer, so we don't worry too much
// about this being in memory only operation
while (token.IsCancellationRequested == false)
{
AsyncManualResetEvent.FrozenAwaiter indexingBatchCompleted;
try
{
indexingBatchCompleted = index.GetIndexingBatchAwaiter();
}
catch (ObjectDisposedException)
{
break;
}
var maxSupersededEtag = 0L;
foreach (var supersededIndex in map.SupersededIndexes)
{
try
{
var etag = supersededIndex.GetLastMappedEtagFor(map.ForCollection);
maxSupersededEtag = Math.Max(etag, maxSupersededEtag);
}
catch (OperationCanceledException)
{
// the superseded index was already deleted
}
}
long currentEtag;
try
{
currentEtag = index.GetLastMappedEtagFor(map.ForCollection);
}
catch (OperationCanceledException)
{
// the index was already disposed by something else
break;
}
if (currentEtag >= maxSupersededEtag)
{
// we'll give it a few seconds to drain any pending queries,
// and because it make it easier to demonstrate how we auto
// clear the old auto indexes.
var timeout = Database.Configuration.Indexing.TimeBeforeDeletionOfSupersededAutoIndex.AsTimeSpan;
if (timeout != TimeSpan.Zero)
{
await TimeoutManager.WaitFor(
timeout
).ConfigureAwait(false);
}
foreach (var supersededIndex in map.SupersededIndexes)
{
try
{
await _indexStore.DeleteIndex(supersededIndex.Name, $"{raftRequestId}/{supersededIndex.Name}");
}
catch (IndexDoesNotExistException)
{
}
}
break;
}
if (await indexingBatchCompleted.WaitAsync() == false)
{
break;
}
}
}
public TermsQueryResultServerSide ExecuteGetTermsQuery(string indexName, string field, string fromValue, long?existingResultEtag, int pageSize, DocumentsOperationContext context, OperationCancelToken token, out Index index)
{
ObjectDisposedException lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
index = GetIndex(indexName);
var etag = index.GetIndexEtag(null);
if (etag == existingResultEtag)
{
return(TermsQueryResultServerSide.NotModifiedResult);
}
return(index.GetTerms(field, fromValue, pageSize, context, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
}
}
throw CreateRetriesFailedException(lastException);
}
protected static void WaitForIndexMap(Index index, long etag)
{
var timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(15);
Assert.True(SpinWait.SpinUntil(() => index.GetLastMappedEtagsForDebug().Values.Min() == etag, timeout));
}
public TermsQueryResultServerSide ExecuteGetTermsQuery(string indexName, string field, string fromValue, long?existingResultEtag, int pageSize, QueryOperationContext queryContext, OperationCancelToken token, out Index index)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
index = GetIndex(indexName);
queryContext.WithIndex(index);
var etag = index.GetIndexEtag(queryContext, null);
if (etag == existingResultEtag)
{
return(TermsQueryResultServerSide.NotModifiedResult);
}
return(index.GetTerms(field, fromValue, pageSize, queryContext, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
WaitForIndexBeingLikelyReplacedDuringQuery().GetAwaiter().GetResult();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
WaitForIndexBeingLikelyReplacedDuringQuery().GetAwaiter().GetResult();
}
}
throw CreateRetriesFailedException(lastException);
}
public TestOperation(Index index, Logger logger) : base(index, logger)
{
}
