public override long GetLastItemEtagInCollection(QueryOperationContext queryContext, string collection)
{
if (collection == Constants.Documents.Collections.AllDocumentsCollection)
{
return(DocumentDatabase.DocumentsStorage.TimeSeriesStorage.GetLastTimeSeriesEtag(queryContext.Documents));
}
return(DocumentDatabase.DocumentsStorage.TimeSeriesStorage.GetLastTimeSeriesEtag(queryContext.Documents, collection));
}
public async Task CanGroupByNestedFieldAndAggregateOnCollection()
{
using (var db = CreateDocumentDatabase())
using (var mri = AutoMapReduceIndex.CreateNew(new AutoMapReduceIndexDefinition(
"Orders",
new[]
{
new AutoIndexField
{
Name = "Lines[].Quantity",
Aggregation = AggregationOperation.Sum,
Storage = FieldStorage.Yes
},
new AutoIndexField
{
Name = "Lines[].Price",
Aggregation = AggregationOperation.Sum,
Storage = FieldStorage.Yes
}
},
new[]
{
new AutoIndexField
{
Name = "ShipTo.Country",
Storage = FieldStorage.Yes
},
}), db))
{
CreateOrders(db, 5, new[] { "Poland", "Israel" });
mri.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);
using (var context = QueryOperationContext.ShortTermSingleUse(db))
{
var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}' WHERE ShipTo.Country = 'Poland'"), context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
var result = queryResult.Results[0].Data;
string location;
Assert.True(result.TryGet("ShipTo.Country", out location));
Assert.Equal("Poland", location);
var price = result["Lines[].Price"] as LazyNumberValue;
Assert.NotNull(price);
Assert.Equal(63.6, price, 1);
var quantity = result["Lines[].Quantity"];
Assert.Equal(9L, quantity);
}
}
}
public GraphQueryPlan(IndexQueryServerSide query, QueryOperationContext context, long?resultEtag,
OperationCancelToken token, DocumentDatabase database)
{
_database = database;
_query = query;
_context = context;
_resultEtag = resultEtag;
_token = token;
_dynamicQueryRunner = new DynamicQueryRunner(database);
}
public CurrentIndexingScope(Index index, DocumentsStorage documentsStorage, QueryOperationContext queryContext, IndexDefinitionBase indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool)
{
_documentsStorage = documentsStorage;
QueryContext = queryContext;
Index = index;
UnmanagedBuffersPool = _unmanagedBuffersPool;
IndexDefinition = indexDefinition;
IndexContext = indexContext;
_getSpatialField = getSpatialField;
}
public override async Task ExecuteStreamQuery(IndexQueryServerSide query, QueryOperationContext queryContext, HttpResponse response, IStreamQueryResultWriter <Document> writer, OperationCancelToken token)
{
var index = GetIndex(query.Metadata.IndexName);
queryContext.WithIndex(index);
using (QueryRunner.MarkQueryAsRunning(index.Name, query, token, true))
{
await index.StreamQuery(response, writer, query, queryContext, token);
}
}
protected override IEnumerable <Reference> GetTombstoneReferences(QueryOperationContext queryContext, CollectionName referencedCollection, long lastEtag, long pageSize)
{
return(_documentsStorage.DocumentDatabase.ServerStore.Cluster.GetCompareExchangeTombstonesByKey(queryContext.Server, _documentsStorage.DocumentDatabase.Name, lastEtag + 1, pageSize)
.Select(x =>
{
_reference.Key = x.Key.StorageKey;
_reference.Etag = x.Index;
return _reference;
}));
}
internal override bool IsStale(QueryOperationContext queryContext, TransactionOperationContext indexContext, long?cutoff = null, long?referenceCutoff = null, long?compareExchangeReferenceCutoff = null, List <string> stalenessReasons = null)
{
var isStale = base.IsStale(queryContext, indexContext, cutoff, referenceCutoff, compareExchangeReferenceCutoff, stalenessReasons);
if (isStale && (stalenessReasons == null || (_handleReferences == null && _handleCompareExchangeReferences == null)))
{
return(isStale);
}
return(StaticIndexHelper.IsStaleDueToReferences(this, queryContext, indexContext, referenceCutoff, compareExchangeReferenceCutoff, stalenessReasons) || isStale);
}
protected override long CalculateIndexEtag(QueryOperationContext queryContext, TransactionOperationContext indexContext, QueryMetadata query, bool isStale)
{
if (_handleReferences == null && _handleCompareExchangeReferences == null)
{
return(base.CalculateIndexEtag(queryContext, indexContext, query, isStale));
}
return(CalculateIndexEtagWithReferences(
_handleReferences, _handleCompareExchangeReferences, queryContext,
indexContext, query, isStale, _referencedCollections, _compiled));
}
public async Task By_single_complex_object()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from user in docs.Users select new {
user.Location,
Count = 1
}" },
Reduce = @"from result in results group result by result.Location into g select new {
Location = g.Key,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Location", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
Put_docs(context.Documents, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal(@"{""Country"":""Poland"",""State"":""Pomerania""}", results[0].Data["Location"].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
}
}
}
}
public async Task Stats()
{
using (var context = QueryOperationContext.Allocate(Database, needsServerContext: true))
{
var stats = new DatabaseStatistics();
FillDatabaseStatistics(stats, context);
await using (var writer = new AsyncBlittableJsonTextWriter(context.Documents, ResponseBodyStream()))
writer.WriteDatabaseStatistics(context.Documents, stats);
}
}
public override async Task ExecuteStreamQuery(IndexQueryServerSide query, QueryOperationContext queryContext, HttpResponse response, IStreamQueryResultWriter <Document> writer,
OperationCancelToken token)
{
var index = await MatchIndex(query, true, customStalenessWaitTimeout : TimeSpan.FromSeconds(60), token.Token);
queryContext.WithIndex(index);
using (QueryRunner.MarkQueryAsRunning(index.Name, query, token, isStreaming: true))
{
await index.StreamQuery(response, writer, query, queryContext, token);
}
}
public async Task Terms()
{
var field = GetQueryStringValueAndAssertIfSingleAndNotEmpty("field");
using (var token = CreateTimeLimitedOperationToken())
using (var context = QueryOperationContext.Allocate(Database))
{
var name = GetIndexNameFromCollectionAndField(field) ?? GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
var fromValue = GetStringQueryString("fromValue", required: false);
var existingResultEtag = GetLongFromHeaders("If-None-Match");
var result = Database.QueryRunner.ExecuteGetTermsQuery(name, field, fromValue, existingResultEtag, GetPageSize(), context, token, out var index);
if (result.NotModified)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return;
}
HttpContext.Response.Headers[Constants.Headers.Etag] = CharExtensions.ToInvariantString(result.ResultEtag);
await using (var writer = new AsyncBlittableJsonTextWriter(context.Documents, ResponseBodyStream()))
{
if (field.EndsWith("__minX") ||
field.EndsWith("__minY") ||
field.EndsWith("__maxX") ||
field.EndsWith("__maxY"))
{
if (index.Definition.IndexFields != null &&
index.Definition.IndexFields.TryGetValue(field.Substring(0, field.Length - 6), out var indexField) == true)
{
if (indexField.Spatial?.Strategy == Client.Documents.Indexes.Spatial.SpatialSearchStrategy.BoundingBox)
{
// Term-values for 'Spatial Index Fields' with 'BoundingBox' are encoded in Lucene as 'prefixCoded bytes'
// Need to convert to numbers for the Studio
var readableTerms = new HashSet <string>();
foreach (var item in result.Terms)
{
var num = Lucene.Net.Util.NumericUtils.PrefixCodedToDouble(item);
readableTerms.Add(NumberUtil.NumberToString(num));
}
result.Terms = readableTerms;
}
}
}
writer.WriteTermsQueryResult(context.Documents, result);
}
}
}
public async Task Progress()
{
using (var context = QueryOperationContext.Allocate(Database, needsServerContext: true))
await using (var writer = new AsyncBlittableJsonTextWriter(context.Documents, ResponseBodyStream()))
using (context.OpenReadTransaction())
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var first = true;
foreach (var index in Database.IndexStore.GetIndexes())
{
try
{
if (index.IsStale(context) == false)
{
continue;
}
var progress = index.GetProgress(context, isStale: true);
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WriteIndexProgress(context.Documents, progress);
}
catch (ObjectDisposedException)
{
// index was deleted
}
catch (OperationCanceledException)
{
// index was deleted
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to get index progress for index name: {index.Name}", e);
}
}
}
writer.WriteEndArray();
writer.WriteEndObject();
}
}
private async Task ServerSideQuery(QueryOperationContext queryContext, RequestTimeTracker tracker, HttpMethod method)
{
var indexQuery = await GetIndexQuery(queryContext.Documents, method, tracker);
await using (var writer = new AsyncBlittableJsonTextWriter(queryContext.Documents, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(indexQuery.ServerSideQuery));
writer.WriteString(indexQuery.ServerSideQuery);
writer.WriteEndObject();
}
}
protected override Gauge32 GetData(DocumentDatabase database)
{
using (var context = QueryOperationContext.Allocate(database, needsServerContext: true))
using (context.OpenReadTransaction())
{
var count = database
.IndexStore
.GetIndexes()
.Count(x => x.IsStale(context));
return(new Gauge32(count));
}
}
protected override IndexingState GetIndexingStateInternal(QueryOperationContext queryContext, TransactionOperationContext indexContext)
{
var result = base.GetIndexingStateInternal(queryContext, indexContext);
if (_handleCompareExchangeReferences == null)
{
return(result);
}
(result.LastProcessedCompareExchangeReferenceEtag, result.LastProcessedCompareExchangeReferenceTombstoneEtag) = StaticIndexHelper.GetLastProcessedCompareExchangeReferenceEtags(this, _compiled, indexContext);
return(result);
}
private static int GetCount(DocumentDatabase database)
{
using (var context = QueryOperationContext.Allocate(database, needsServerContext: true))
using (context.OpenReadTransaction())
{
var count = database
.IndexStore
.GetIndexes()
.Count(x => x.IsStale(context));
return(count);
}
}
public override async Task ExecuteStreamQuery(IndexQueryServerSide query, QueryOperationContext queryContext, HttpResponse response, IStreamQueryResultWriter <Document> writer, OperationCancelToken token)
{
using (var context = QueryOperationContext.Allocate(Database, needsServerContext: false))
{
var result = new StreamDocumentQueryResult(response, writer, token)
{
IndexName = Constants.Documents.Indexing.DummyGraphIndexName
};
result = await ExecuteQuery(result, query, context, null, token);
result.Flush();
}
}
public CanContinueBatchParameters(IndexingStatsScope stats, IndexingWorkType workType, QueryOperationContext queryContext, TransactionOperationContext indexingContext,
Lazy <IndexWriteOperation> indexWriteOperation, long currentEtag, long maxEtag, long count,
Stopwatch sw)
{
Stats = stats;
WorkType = workType;
QueryContext = queryContext;
IndexingContext = indexingContext;
IndexWriteOperation = indexWriteOperation;
CurrentEtag = currentEtag;
MaxEtag = maxEtag;
Count = count;
Sw = sw;
}
public async Task PatchByIndex_WhenFinish_ShouldFreeInternalUsageMemory()
{
using (var store = GetDocumentStore())
{
var database = await GetDatabase(store.Database);
var index = new IndexDefinition
{
Name = "Users_ByName",
Maps = { "from user in docs.Users select new { user.Name }" },
Type = IndexType.Map
};
await store
.Maintenance
.SendAsync(new PutIndexesOperation(new[] { index }));
using (var session = store.OpenAsyncSession())
{
for (var i = 0; i < 100; i++)
{
await session.StoreAsync(new User { Name = "John" });
}
await session.SaveChangesAsync();
}
WaitForIndexing(store);
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
var query = new IndexQueryServerSide($"FROM index '{index.Name}'");
var patch = new PatchRequest("var u = this; u.is = true;", PatchRequestType.Patch, query.Metadata.DeclaredFunctions);
var before = context.Documents.AllocatedMemory;
await database.QueryRunner.ExecutePatchQuery(
query,
new QueryOperationOptions { RetrieveDetails = true },
patch,
query.QueryParameters,
context,
p => { },
new OperationCancelToken(CancelAfter, CancellationToken.None, CancellationToken.None));
var after = context.Documents.AllocatedMemory;
//In a case of fragmentation, we don't immediately freeing memory so the memory can be a little bit higher
const long threshold = 256;
Assert.True(Math.Abs(before - after) < threshold);
}
}
}
private async Task DetailedGraphResult(QueryOperationContext queryContext, RequestTimeTracker tracker, HttpMethod method)
{
var indexQuery = await GetIndexQuery(queryContext.Documents, method, tracker);
var queryRunner = Database.QueryRunner.GetRunner(indexQuery);
if (!(queryRunner is GraphQueryRunner gqr))
{
throw new InvalidOperationException("The specified query is not a graph query.");
}
using (var token = CreateTimeLimitedQueryToken())
await using (var writer = new AsyncBlittableJsonTextWriter(queryContext.Documents, ResponseBodyStream()))
{
await gqr.WriteDetailedQueryResult(indexQuery, queryContext, writer, token);
}
}
public async Task MultipleReduceKeys(int numberOfUsers, string[] locations)
{
using (var db = CreateDocumentDatabase())
using (var index = AutoMapReduceIndex.CreateNew(GetUsersCountByLocationIndexDefinition(), db))
{
CreateUsers(db, numberOfUsers, locations);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
while (index.DoIndexingWork(scope, cts.Token))
{
;
}
Assert.Equal(numberOfUsers, batchStats.MapAttempts);
Assert.Equal(numberOfUsers, batchStats.MapSuccesses);
Assert.Equal(0, batchStats.MapErrors);
Assert.True(batchStats.ReduceAttempts >= numberOfUsers, $"{batchStats.ReduceAttempts} >= {numberOfUsers}");
Assert.True(batchStats.ReduceSuccesses >= numberOfUsers, $"{batchStats.ReduceSuccesses} >= {numberOfUsers}");
Assert.Equal(batchStats.ReduceAttempts, batchStats.ReduceSuccesses);
Assert.Equal(0, batchStats.ReduceErrors);
using (var context = QueryOperationContext.ShortTermSingleUse(db))
{
var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'")
{
WaitForNonStaleResultsTimeout = TimeSpan.FromMinutes(1)
}, context, OperationCancelToken.None);
Assert.False(queryResult.IsStale);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected, results[i].Data["Count"]);
}
}
}
}
public async Task HandleQuery(HttpMethod httpMethod)
{
using (var tracker = new RequestTimeTracker(HttpContext, Logger, Database, "Query"))
{
try
{
using (var token = CreateTimeLimitedQueryToken())
using (var queryContext = QueryOperationContext.Allocate(Database))
{
var debug = GetStringQueryString("debug", required: false);
if (string.IsNullOrWhiteSpace(debug) == false)
{
await Debug(queryContext, debug, token, tracker, httpMethod);
return;
}
var diagnostics = GetBoolValueQueryString("diagnostics", required: false) ?? false;
await Query(queryContext, token, tracker, httpMethod, diagnostics);
}
}
catch (Exception e)
{
if (tracker.Query == null)
{
string errorMessage;
if (e is EndOfStreamException || e is ArgumentException)
{
errorMessage = "Failed: " + e.Message;
}
else
{
errorMessage = "Failed: " +
HttpContext.Request.Path.Value +
e.ToString();
}
tracker.Query = errorMessage;
if (TrafficWatchManager.HasRegisteredClients)
{
AddStringToHttpContext(errorMessage, TrafficWatchChangeType.Queries);
}
}
throw;
}
}
}
private async Task Explain(QueryOperationContext queryContext, RequestTimeTracker tracker, HttpMethod method)
{
var indexQuery = await GetIndexQuery(queryContext.Documents, method, tracker);
var explanations = Database.QueryRunner.ExplainDynamicIndexSelection(indexQuery, out string indexName);
await using (var writer = new AsyncBlittableJsonTextWriter(queryContext.Documents, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("IndexName");
writer.WriteString(indexName);
writer.WriteComma();
writer.WriteArray(queryContext.Documents, "Results", explanations, (w, c, explanation) => w.WriteExplanation(queryContext.Documents, explanation));
writer.WriteEndObject();
}
}
protected async Task <SuggestionQueryResult> ExecuteSuggestion(
IndexQueryServerSide query,
Index index,
QueryOperationContext queryContext,
long?existingResultEtag,
OperationCancelToken token)
{
if (query.Metadata.SelectFields.Length == 0)
{
throw new InvalidQueryException("Suggestion query must have at least one suggest token in SELECT.", query.Metadata.QueryText, query.QueryParameters);
}
var fields = index.Definition.IndexFields;
foreach (var f in query.Metadata.SelectFields)
{
if (f.IsSuggest == false)
{
throw new InvalidQueryException("Suggestion query must have only suggest tokens in SELECT.", query.Metadata.QueryText, query.QueryParameters);
}
var selectField = (SuggestionField)f;
if (fields.TryGetValue(selectField.Name, out var field) == false)
{
throw new InvalidOperationException($"Index '{index.Name}' does not have a field '{selectField.Name}'.");
}
if (field.HasSuggestions == false)
{
throw new InvalidOperationException($"Index '{index.Name}' does not have suggestions configured for field '{selectField.Name}'.");
}
}
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag(queryContext, query.Metadata);
if (etag == existingResultEtag.Value)
{
return(SuggestionQueryResult.NotModifiedResult);
}
}
return(await index.SuggestionQuery(query, queryContext, token));
}
public override bool Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var moreWorkFound = base.Execute(queryContext, indexContext, writeOperation, stats, token);
if (_mapReduceIndex.OutputReduceToCollection?.HasDocumentsToDelete(indexContext) == true)
{
moreWorkFound = true;
if (_mapReduceIndex.IsSideBySide() == false)
{
// we can start deleting reduce output documents only if index becomes a regular one
moreWorkFound |= _mapReduceIndex.OutputReduceToCollection.DeleteDocuments(stats, indexContext);
}
}
return(moreWorkFound);
}
private static void FillIndexInfo(Index index, QueryOperationContext context, DateTime now, DatabaseStatusReport report)
{
var stats = index.GetIndexingState(context);
var lastQueried = GetLastQueryInfo(index, now);
//We might have old version of this index with the same name
report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
LastIndexedCompareExchangeReferenceEtag = stats.LastProcessedCompareExchangeReferenceEtag,
LastIndexedCompareExchangeReferenceTombstoneEtag = stats.LastProcessedCompareExchangeReferenceTombstoneEtag,
LastQueried = lastQueried,
IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase),
IsStale = stats.IsStale,
State = index.State,
LastTransactionId = index.LastTransactionId
};
}
public override async Task ExecuteStreamQuery(IndexQueryServerSide query, QueryOperationContext queryContext, HttpResponse response, IStreamQueryResultWriter <Document> writer, OperationCancelToken token)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
await GetRunner(query).ExecuteStreamQuery(query, queryContext, response, writer, token);
return;
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public async Task Should_throw_on_attempt_to_create_auto_index()
{
using (var database = CreateDocumentDatabase())
{
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
var ex = await Assert.ThrowsAsync <InvalidOperationException>(async() =>
{
await database.QueryRunner.ExecuteQuery(new IndexQueryServerSide("from Users where LastName = 'Arek'")
{
DisableAutoIndexCreation = true
}, context, null,
OperationCancelToken.None);
});
Assert.Equal("Creation of Auto Indexes was disabled and no Auto Index matching the given query was found.", ex.Message);
}
}
}
internal override bool IsStale(QueryOperationContext queryContext, TransactionOperationContext indexContext, long?cutoff = null, long?referenceCutoff = null, long?compareExchangeReferenceCutoff = null, List <string> stalenessReasons = null)
{
var isStale = base.IsStale(queryContext, indexContext, cutoff, referenceCutoff, compareExchangeReferenceCutoff, stalenessReasons);
if (isStale == false && OutputReduceToCollection?.HasDocumentsToDelete(indexContext) == true)
{
if (indexContext.IgnoreStalenessDueToReduceOutputsToDelete == false)
{
isStale = true;
stalenessReasons?.Add($"There are still some reduce output documents to delete from collection '{Definition.OutputReduceToCollection}'. ");
}
}
if (isStale && (stalenessReasons == null || (_handleReferences == null && _handleCompareExchangeReferences == null)))
{
return(isStale);
}
return(StaticIndexHelper.IsStaleDueToReferences(this, queryContext, indexContext, referenceCutoff, compareExchangeReferenceCutoff, stalenessReasons) || isStale);
}
